fixed default value

Table meta support

.gitignore

@@ -13,3 +13,4 @@ otter.properties
 jtester.properties
 .idea/
 *.iml
+.DS_Store

README.md

@@ -9,7 +9,7 @@
 
 <h1>背景</h1>
 <p style="font-size: 14px;">   早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&amp;消费的业务，从此开启了一段新纪元。</p>
-<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.6及以下的版本(阿里内部mysql 5.6.10, mysql 5.5.18和5.1.40/48)</p>
+<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.7及以下的版本(阿里内部mysql 5.7.13, 5.6.10, mysql 5.5.18和5.1.40/48)</p>
 <p style="font-size: 14px;"> </p>
 <p style="font-size: 14px;">基于日志增量订阅&amp;消费支持的业务：</p>
 <ol style="font-size: 14px;">
@@ -36,7 +36,7 @@
 <li>slave重做中继日志中的事件，将改变反映它自己的数据。</li>
 </ol>
 <h3>canal的工作原理：</h3>
-<p><img width="590" src="http://dl.iteye.com/upload/attachment/0080/3107/c87b67ba-394c-3086-9577-9db05be04c95.jpg" alt="" height="273">
+<p><img width="590" src="https://camo.githubusercontent.com/46c626b4cde399db43b2634a7911a04aecf273a0/687474703a2f2f646c2e69746579652e636f6d2f75706c6f61642f6174746163686d656e742f303038302f333130372f63383762363762612d333934632d333038362d393537372d3964623035626530346339352e6a7067" alt="" height="273">
 <p>原理相对比较简单：</p>
 <ol>
 <li>canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议</li>
@@ -62,6 +62,17 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
 </ul>
 
+<h1>相关资料</h1>
+
+* ADC阿里技术嘉年华分享ppt (放在google docs上，可能需要翻墙): <a href="https://docs.google.com/presentation/d/1MkszUPYRDkfVPz9IqOT1LLT5d9tuwde_WC8GZvjaDRg/edit?usp=sharing">ppt下载</href>  
+* [与阿里巴巴的RocketMQ配合使用](https://github.com/alibaba/RocketMQ)
+
+<h1>相关开源</h1>
+<ol>
+<li>阿里巴巴分布式数据库同步系统(解决中美异地机房)：<a href="http://github.com/alibaba/otter">http://github.com/alibaba/otter</a></li>
+<li>阿里巴巴去Oracle数据迁移同步工具(目标支持MySQL/DRDS)：<a href="http://github.com/alibaba/yugong">http://github.com/alibaba/yugong</a></li>
+</ol>
+
 <h1>问题反馈</h1>
 <ol>
 <li>qq交流群： 161559791 </li>
@@ -69,3 +80,10 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li>新浪微博： agapple0002 </li>
 <li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
 </ol>
+
+<pre>
+【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
+岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
+工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊. 
+具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
+</pre>

client/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java

@@ -19,14 +19,14 @@ public interface CanalConnector {
      * 
      * @throws CanalClientException
      */
-    public void connect() throws CanalClientException;
+    void connect() throws CanalClientException;
 
     /**
      * 释放链接
      * 
      * @throws CanalClientException
      */
-    public void disconnect() throws CanalClientException;
+    void disconnect() throws CanalClientException;
 
     /**
      * 检查下链接是否合法
@@ -43,7 +43,7 @@ public interface CanalConnector {
      * 
      * @throws CanalClientException
      */
-    public boolean checkValid() throws CanalClientException;
+    boolean checkValid() throws CanalClientException;
 
     /**
      * 客户端订阅，重复订阅时会更新对应的filter信息
@@ -56,7 +56,6 @@ public interface CanalConnector {
      * TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
      * </pre>
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void subscribe(String filter) throws CanalClientException;
@@ -64,7 +63,6 @@ public interface CanalConnector {
     /**
      * 客户端订阅，不提交客户端filter，以服务端的filter为准
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void subscribe() throws CanalClientException;
@@ -72,7 +70,6 @@ public interface CanalConnector {
     /**
      * 取消订阅
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void unsubscribe() throws CanalClientException;
@@ -140,14 +137,14 @@ public interface CanalConnector {
     void ack(long batchId) throws CanalClientException;
 
     /**
-     * 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId
+     * 回滚到未进行 {@link #ack} 的地方，指定回滚具体的batchId
      * 
      * @throws CanalClientException
      */
     void rollback(long batchId) throws CanalClientException;
 
     /**
-     * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
+     * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
      * 
      * @throws CanalClientException
      */

client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java

@@ -21,6 +21,7 @@ public class CanalConnectors {
      * 创建单链接的客户端链接
      *
      * @param address
+     * @param destination
      * @param username
      * @param password
      * @return
@@ -36,13 +37,16 @@ public class CanalConnectors {
      * 创建带cluster模式的客户端链接，自动完成failover切换
      *
      * @param addresses
+     * @param destination
      * @param username
      * @param password
      * @return
      */
     public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
                                                      String username, String password) {
-        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
+            password,
+            destination,
             new SimpleNodeAccessStrategy(addresses));
         canalConnector.setSoTimeout(30 * 1000);
         return canalConnector;
@@ -51,19 +55,18 @@ public class CanalConnectors {
     /**
      * 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描
      *
+     * @param zkServers
+     * @param destination
      * @param username
      * @param password
      * @return
      */
     public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
                                                      String password) {
-        ClusterCanalConnector canalConnector = new ClusterCanalConnector(
-                                                                         username,
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
             password,
             destination,
-                                                                         new ClusterNodeAccessStrategy(
-                                                                                                       destination,
-                                                                                                       ZkClientx.getZkClient(zkServers)));
+            new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkServers)));
         canalConnector.setSoTimeout(30 * 1000);
         return canalConnector;
     }

client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java

@@ -10,5 +10,7 @@ import java.net.SocketAddress;
  */
 public interface CanalNodeAccessStrategy {
 
+    SocketAddress currentNode();
+
     SocketAddress nextNode();
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.client.impl;
 import java.net.SocketAddress;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,11 +40,17 @@ public class ClusterCanalConnector implements CanalConnector {
 
     public void connect() throws CanalClientException {
         while (currentConnector == null) {
-            SocketAddress nextAddress = this.accessStrategy.nextNode();
             int times = 0;
             while (true) {
                 try {
-                    currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination);
+                    currentConnector = new SimpleCanalConnector(null, username, password, destination) {
+
+                        @Override
+                        public SocketAddress getNextAddress() {
+                            return accessStrategy.nextNode();
+                        }
+
+                    };
                     currentConnector.setSoTimeout(soTimeout);
                     if (filter != null) {
                         currentConnector.setFilter(filter);
@@ -57,7 +62,7 @@ public class ClusterCanalConnector implements CanalConnector {
                     currentConnector.connect();
                     break;
                 } catch (Exception e) {
-                    logger.warn("failed to connect to:{} after retry {} times", nextAddress, times);
+                    logger.warn("failed to connect to:{} after retry {} times", accessStrategy.currentNode(), times);
                     currentConnector.disconnect();
                     currentConnector = null;
                     // retry for #retryTimes for each node when trying to
@@ -101,9 +106,10 @@ public class ClusterCanalConnector implements CanalConnector {
                 this.filter = filter;
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when subscribing from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format(
+                        "something goes wrong when subscribing from server: %s",
+                        currentConnector != null ? currentConnector.getAddress() : "null"),
+                        t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -120,9 +126,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.unsubscribe();
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when unsubscribing from server:%s",
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -138,9 +143,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.get(batchSize);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getting data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when getting data from server:%s",
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -156,9 +160,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.get(batchSize, timeout, unit);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getting data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when getting data from server:%s",
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -174,9 +177,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.getWithoutAck(batchSize);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
+                    currentConnector.getAddress()), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -192,9 +194,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
+                    currentConnector.getAddress()), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -210,9 +211,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.rollback(batchId);
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
+                    currentConnector.getAddress()), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -228,9 +228,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.rollback();
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
+                    currentConnector.getAddress()), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -247,9 +246,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.ack(batchId);
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when acking data from server:{}\n{}",
-                    currentConnector.getAddress(),
-                    ExceptionUtils.getFullStackTrace(t));
+                logger.warn(String.format("something goes wrong when acking data from server:%s",
+                    currentConnector.getAddress()), t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java

@@ -62,6 +62,11 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
         initRunning(this.zkClient.readData(runningPath, true));
     }
 
+    @Override
+    public SocketAddress currentNode() {
+        return nextNode();
+    }
+
     public SocketAddress nextNode() {
         if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点
             return runningAddress;

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java

@@ -5,7 +5,11 @@ import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.SocketChannel;
+import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -37,7 +41,6 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 
 /**
  * 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作
@@ -57,6 +60,8 @@ public class SimpleCanalConnector implements CanalConnector {
     private final ByteBuffer     readHeader            = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
     private final ByteBuffer     writeHeader           = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
     private SocketChannel        channel;
+    private ReadableByteChannel  readableChannel;
+    private WritableByteChannel  writableChannel;
     private List<Compression>    supportedCompressions = new ArrayList<Compression>();
     private ClientIdentity       clientIdentity;
     private ClientRunningMonitor runningMonitor;                                                             // 运行控制
@@ -125,8 +130,14 @@ public class SimpleCanalConnector implements CanalConnector {
         try {
             channel = SocketChannel.open();
             channel.socket().setSoTimeout(soTimeout);
+            SocketAddress address = getAddress();
+            if (address == null) {
+                address = getNextAddress();
+            }
             channel.connect(address);
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            readableChannel = Channels.newChannel(channel.socket().getInputStream());
+            writableChannel = Channels.newChannel(channel.socket().getOutputStream());
+            Packet p = Packet.parseFrom(readNextPacket());
             if (p.getVersion() != 1) {
                 throw new CanalClientException("unsupported version at this client.");
             }
@@ -140,17 +151,17 @@ public class SimpleCanalConnector implements CanalConnector {
             //
             ClientAuth ca = ClientAuth.newBuilder()
                 .setUsername(username != null ? username : "")
+                .setPassword(ByteString.copyFromUtf8(password != null ? password : ""))
                 .setNetReadTimeout(soTimeout)
                 .setNetWriteTimeout(soTimeout)
                 .build();
-            writeWithHeader(channel,
-                Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTAUTHENTICATION)
                 .setBody(ca.toByteString())
                 .build()
                 .toByteArray());
             //
-            Packet ack = Packet.parseFrom(readNextPacket(channel));
+            Packet ack = Packet.parseFrom(readNextPacket());
             if (ack.getType() != PacketType.ACK) {
                 throw new CanalClientException("unexpected packet type when ack is expected");
             }
@@ -169,14 +180,26 @@ public class SimpleCanalConnector implements CanalConnector {
     }
 
     private void doDisconnnect() throws CanalClientException {
+        if (readableChannel != null) {
+            quietlyClose(readableChannel);
+            readableChannel = null;
+        }
+        if (writableChannel != null) {
+            quietlyClose(writableChannel);
+            writableChannel = null;
+        }
         if (channel != null) {
+            quietlyClose(channel);
+            channel = null;
+        }
+    }
+
+    private void quietlyClose(Channel channel) {
         try {
             channel.close();
         } catch (IOException e) {
             logger.warn("exception on closing channel:{} \n {}", channel, e);
         }
-            channel = null;
-        }
     }
 
     public void subscribe() throws CanalClientException {
@@ -186,8 +209,7 @@ public class SimpleCanalConnector implements CanalConnector {
     public void subscribe(String filter) throws CanalClientException {
         waitClientRunning();
         try {
-            writeWithHeader(channel,
-                Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.SUBSCRIPTION)
                 .setBody(Sub.newBuilder()
                     .setDestination(clientIdentity.getDestination())
@@ -198,7 +220,7 @@ public class SimpleCanalConnector implements CanalConnector {
                 .build()
                 .toByteArray());
             //
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Packet p = Packet.parseFrom(readNextPacket());
             Ack ack = Ack.parseFrom(p.getBody());
             if (ack.getErrorCode() > 0) {
                 throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
@@ -213,8 +235,7 @@ public class SimpleCanalConnector implements CanalConnector {
     public void unsubscribe() throws CanalClientException {
         waitClientRunning();
         try {
-            writeWithHeader(channel,
-                Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.UNSUBSCRIPTION)
                 .setBody(Unsub.newBuilder()
                     .setDestination(clientIdentity.getDestination())
@@ -224,7 +245,7 @@ public class SimpleCanalConnector implements CanalConnector {
                 .build()
                 .toByteArray());
             //
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Packet p = Packet.parseFrom(readNextPacket());
             Ack ack = Ack.parseFrom(p.getBody());
             if (ack.getErrorCode() > 0) {
                 throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
@@ -257,8 +278,7 @@ public class SimpleCanalConnector implements CanalConnector {
                 unit = TimeUnit.MILLISECONDS;
             }
 
-            writeWithHeader(channel,
-                Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.GET)
                 .setBody(Get.newBuilder()
                     .setAutoAck(false)
@@ -271,15 +291,14 @@ public class SimpleCanalConnector implements CanalConnector {
                     .toByteString())
                 .build()
                 .toByteArray());
-
             return receiveMessages();
         } catch (IOException e) {
             throw new CanalClientException(e);
         }
     }
 
-    private Message receiveMessages() throws InvalidProtocolBufferException, IOException {
-        Packet p = Packet.parseFrom(readNextPacket(channel));
+    private Message receiveMessages() throws IOException {
+        Packet p = Packet.parseFrom(readNextPacket());
         switch (p.getType()) {
             case MESSAGES: {
                 if (!p.getCompression().equals(Compression.NONE)) {
@@ -311,7 +330,7 @@ public class SimpleCanalConnector implements CanalConnector {
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(channel, Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTACK)
                 .setBody(ca.toByteString())
                 .build()
@@ -329,7 +348,7 @@ public class SimpleCanalConnector implements CanalConnector {
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(channel, Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTROLLBACK)
                 .setBody(ca.toByteString())
                 .build()
@@ -346,7 +365,15 @@ public class SimpleCanalConnector implements CanalConnector {
 
     // ==================== helper method ====================
 
-    private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
+    private void writeWithHeader(byte[] body) throws IOException {
+        writeWithHeader(writableChannel, body);
+    }
+
+    private byte[] readNextPacket() throws IOException {
+        return readNextPacket(readableChannel);
+    }
+
+    private void writeWithHeader(WritableByteChannel channel, byte[] body) throws IOException {
         synchronized (writeDataLock) {
             writeHeader.clear();
             writeHeader.putInt(body.length);
@@ -356,7 +383,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private byte[] readNextPacket(SocketChannel channel) throws IOException {
+    private byte[] readNextPacket(ReadableByteChannel channel) throws IOException {
         synchronized (readDataLock) {
             readHeader.clear();
             read(channel, readHeader);
@@ -367,7 +394,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
+    private void read(ReadableByteChannel channel, ByteBuffer buffer) throws IOException {
         while (buffer.hasRemaining()) {
             int r = channel.read(buffer);
             if (r == -1) {
@@ -411,7 +438,6 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-
     private void waitClientRunning() {
         try {
             if (zkClientx != null) {
@@ -435,6 +461,10 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
+    public SocketAddress getNextAddress() {
+        return null;
+    }
+
     public SocketAddress getAddress() {
         return address;
     }

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java

@@ -32,4 +32,9 @@ public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
         }
     }
 
+    @Override
+    public SocketAddress currentNode() {
+        return nodes.get(index);
+    }
+
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.client.impl.running;
 
 import java.net.InetSocketAddress;
+import java.text.MessageFormat;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -20,6 +21,7 @@ import com.alibaba.otter.canal.common.utils.BooleanMutex;
 import com.alibaba.otter.canal.common.utils.JsonUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 
 /**
  * clinet running控制
@@ -62,6 +64,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
             public void handleDataDeleted(String dataPath) throws Exception {
                 MDC.put("destination", destination);
                 mutex.set(false);
+                // 触发一下退出,可能是人为干预的释放操作或者网络闪断引起的session expired timeout
+                processActiveExit();
                 if (!release && activeData != null && isMine(activeData.getAddress())) {
                     // 如果上一次active的状态就是本机，则即时触发一下active抢占
                     initRunning();
@@ -96,7 +100,11 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
         releaseRunning(); // 尝试一下release
     }
 
-    public void initRunning() {
+    // 改动记录：
+    // 1,在方法上加synchronized关键字,保证同步顺序执行;
+    // 2,判断Zk上已经存在的activeData是否是本机，是的话把mutex重置为true，否则会导致死锁
+    // 3,增加异常处理，保证出现异常时，running节点能被删除,否则会导致死锁
+    public synchronized void initRunning() {
         if (!isStart()) {
             return;
         }
@@ -116,11 +124,22 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
                 initRunning();
             } else {
                 activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+                // 如果发现已经存在,判断一下是否自己,避免活锁
+                if (activeData.getAddress().contains(":") && isMine(activeData.getAddress())) {
+                    mutex.set(true);
+                }
             }
         } catch (ZkNoNodeException e) {
             zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()),
                 true); // 尝试创建父节点
             initRunning();
+        } catch (Throwable t) {
+            logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].",
+                destination),
+                t);
+            // 出现任何异常尝试release
+            releaseRunning();
+            throw new CanalClientException("something goes wrong in initRunning method. ", t);
         }
     }
 

client/src/test/java/logback.xml

@@ -8,7 +8,7 @@
 		</encoder>
 	</appender>
 	
-	<root level="WARN">
+	<root level="INFO">
 		<appender-ref ref="STDOUT"/>
 	</root>
 </configuration>

codeformat.xml

@@ -0,0 +1,291 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<profiles version="12">
+<profile kind="CodeFormatterProfile" name="canal-format" version="12">
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
+<setting id="org.eclipse.jdt.core.compiler.source" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="120"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="true"/>
+</profile>
+</profiles>

codetemplates.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><templates><template autoinsert="true" context="gettercomment_context" deleted="false" description="Comment for getter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name="gettercomment">/**
+ * @return the ${bare_field_name}
+ */</template><template autoinsert="true" context="settercomment_context" deleted="false" description="Comment for setter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.settercomment" name="settercomment">/**
+ * @param ${param} the ${bare_field_name} to set
+ */</template><template autoinsert="true" context="constructorcomment_context" deleted="false" description="Comment for created constructors" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name="constructorcomment">/**
+ * ${tags}
+ */</template><template autoinsert="false" context="filecomment_context" deleted="false" description="Comment for created Java files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.filecomment" name="filecomment"/><template autoinsert="false" context="typecomment_context" deleted="false" description="Comment for created types" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.typecomment" name="typecomment">/**
+ * @author ${user} ${date} ${time}
+ * @since 1.0.0
+ */</template><template autoinsert="true" context="fieldcomment_context" deleted="false" description="Comment for fields" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name="fieldcomment">/**
+ * 
+ */</template><template autoinsert="true" context="methodcomment_context" deleted="false" description="Comment for non-overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name="methodcomment">/**
+ * ${tags}
+ */</template><template autoinsert="false" context="overridecomment_context" deleted="false" description="Comment for overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name="overridecomment"/><template autoinsert="true" context="delegatecomment_context" deleted="false" description="Comment for delegate methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name="delegatecomment">/**
+ * ${tags}
+ * ${see_to_target}
+ */</template><template autoinsert="false" context="newtype_context" deleted="false" description="Newly created files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.newtype" name="newtype">${filecomment}
+${package_declaration}
+${typecomment}
+${type_declaration}</template><template autoinsert="true" context="classbody_context" deleted="false" description="Code in new class type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.classbody" name="classbody">
+</template><template autoinsert="true" context="interfacebody_context" deleted="false" description="Code in new interface type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name="interfacebody">
+</template><template autoinsert="true" context="enumbody_context" deleted="false" description="Code in new enum type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.enumbody" name="enumbody">
+</template><template autoinsert="true" context="annotationbody_context" deleted="false" description="Code in new annotation type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name="annotationbody">
+</template><template autoinsert="true" context="catchblock_context" deleted="false" description="Code in new catch blocks" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.catchblock" name="catchblock">// ${todo} Auto-generated catch block
+${exception_var}.printStackTrace();</template><template autoinsert="true" context="methodbody_context" deleted="false" description="Code in created method stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodbody" name="methodbody">// ${todo} Auto-generated method stub
+${body_statement}</template><template autoinsert="true" context="constructorbody_context" deleted="false" description="Code in created constructor stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name="constructorbody">${body_statement}
+// ${todo} Auto-generated constructor stub</template><template autoinsert="true" context="getterbody_context" deleted="false" description="Code in created getters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.getterbody" name="getterbody">return ${field};</template><template autoinsert="true" context="setterbody_context" deleted="false" description="Code in created setters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.setterbody" name="setterbody">${field} = ${param};</template></templates>

common/pom.xml

@@ -1,24 +1,29 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
-	<groupId>com.alibaba.otter</groupId>
 	<artifactId>canal.common</artifactId>
 	<packaging>jar</packaging>
 	<name>canal common module for otter ${project.version}</name>
 	<url>http://b2b-doc.alibaba-inc.com/display/opentech/Otter</url>
 	<dependencies>
+		<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
+		<dependency>
+			<groupId>io.netty</groupId>
+			<artifactId>netty-all</artifactId>
+		</dependency>
 		<!-- zk -->
 		<dependency>
 			<groupId>org.apache.zookeeper</groupId>
 			<artifactId>zookeeper</artifactId>
 		</dependency>
 		<dependency>
-			<groupId>com.github.sgroschupf</groupId>
+			<groupId>com.101tec</groupId>
 			<artifactId>zkclient</artifactId>
 		</dependency>
 		<!-- external -->
@@ -29,11 +34,10 @@
 		<dependency>
 			<groupId>commons-lang</groupId>
 			<artifactId>commons-lang</artifactId>
-			<version>2.6</version>
 		</dependency>
 		<dependency>
-			<groupId>org.springframework</groupId>
-			<artifactId>spring</artifactId>
+			<groupId>commons-codec</groupId>
+			<artifactId>commons-codec</artifactId>
 		</dependency>
 		<dependency>
 			<groupId>com.alibaba</groupId>
@@ -60,10 +64,37 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-aop</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-context</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-jdbc</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-orm</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-test</artifactId>
+			<version>${spring_version}</version>
+			<scope>test</scope>
+		</dependency>
 		<!-- junit -->
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
+			<scope>test</scope>
 		</dependency>
 	</dependencies>
 </project>

common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java

@@ -6,9 +6,9 @@ package com.alibaba.otter.canal.common;
  */
 public interface CanalLifeCycle {
 
-    public void start();
+    void start();
 
-    public void stop();
+    void stop();
 
-    public boolean isStart();
+    boolean isStart();
 }

common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java

@@ -14,8 +14,7 @@ public interface CanalAlarmHandler extends CanalLifeCycle {
      * 发送对应destination的报警
      * 
      * @param destination
-     * @param title
      * @param msg
      */
-    public void sendAlarm(String destination, String msg);
+    void sendAlarm(String destination, String msg);
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java

@@ -38,7 +38,8 @@ public class AddressUtils {
     private static boolean isValidHostAddress(InetAddress address) {
         if (address == null || address.isLoopbackAddress()) return false;
         String name = address.getHostAddress();
-        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches());
+        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name)
+            .matches());
     }
 
     public static String getHostIp() {

common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java

@@ -1,12 +1,19 @@
 package com.alibaba.otter.canal.common.utils;
 
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
 import java.util.Arrays;
 import java.util.List;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.TypeReference;
 import com.alibaba.fastjson.serializer.JSONSerializer;
+import com.alibaba.fastjson.serializer.ObjectSerializer;
 import com.alibaba.fastjson.serializer.PropertyFilter;
+import com.alibaba.fastjson.serializer.SerializeConfig;
 import com.alibaba.fastjson.serializer.SerializeWriter;
 import com.alibaba.fastjson.serializer.SerializerFeature;
 
@@ -17,6 +24,12 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
  */
 public class JsonUtils {
 
+    static {
+        SerializeConfig.getGlobalInstance().put(InetAddress.class, InetAddressSerializer.instance);
+        SerializeConfig.getGlobalInstance().put(Inet4Address.class, InetAddressSerializer.instance);
+        SerializeConfig.getGlobalInstance().put(Inet6Address.class, InetAddressSerializer.instance);
+    }
+
     public static <T> T unmarshalFromByte(byte[] bytes, Class<T> targetClass) {
         return (T) JSON.parseObject(bytes, targetClass);// 默认为UTF-8
     }
@@ -70,4 +83,22 @@ public class JsonUtils {
             out.close();
         }
     }
+
+    public static class InetAddressSerializer implements ObjectSerializer {
+
+        public static InetAddressSerializer instance = new InetAddressSerializer();
+
+        @Override
+        public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features)
+                                                                                                                   throws IOException {
+            if (object == null) {
+                serializer.writeNull();
+                return;
+            }
+
+            InetAddress address = (InetAddress) object;
+            // 优先使用name
+            serializer.write(address.getHostName());
+        }
+    }
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java

@@ -54,6 +54,7 @@ public final class UriUtils {
         }
         String query = uri.getRawQuery();
         HashMap<String, String> params = new HashMap<String, String>();
+        @SuppressWarnings("resource")
         Scanner scan = new Scanner(query);
         scan.useDelimiter(SPLIT);
         while (scan.hasNext()) {

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java

@@ -12,7 +12,7 @@ import org.I0Itec.zkclient.serialize.ZkSerializer;
 import org.apache.zookeeper.CreateMode;
 
 import com.google.common.base.Function;
-import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * 使用自定义的ZooKeeperx for zk connection
@@ -23,7 +23,7 @@ import com.google.common.collect.MapMaker;
 public class ZkClientx extends ZkClient {
 
     // 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
-    private static Map<String, ZkClientx> clients = new MapMaker().makeComputingMap(new Function<String, ZkClientx>() {
+    private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>() {
 
                                                       public ZkClientx apply(String servers) {
                                                           return new ZkClientx(servers);
@@ -62,10 +62,12 @@ public class ZkClientx extends ZkClient {
      * Create a persistent Sequential node.
      * 
      * @param path
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
-     * in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * @param createParents if true all parent dirs are created as well and no
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
+     * @throws ZkInterruptedException if operation was interrupted, or a
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */
@@ -89,10 +91,12 @@ public class ZkClientx extends ZkClient {
      * 
      * @param path
      * @param data
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
-     * in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * @param createParents if true all parent dirs are created as well and no
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
+     * @throws ZkInterruptedException if operation was interrupted, or a
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */
@@ -118,10 +122,12 @@ public class ZkClientx extends ZkClient {
      * 
      * @param path
      * @param data
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
-     * in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * @param createParents if true all parent dirs are created as well and no
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
+     * @throws ZkInterruptedException if operation was interrupted, or a
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java

@@ -8,20 +8,15 @@ import java.util.List;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.I0Itec.zkclient.IZkConnection;
+import org.I0Itec.zkclient.ZkConnection;
 import org.I0Itec.zkclient.exception.ZkException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zookeeper.ClientCnxn;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.client.ConnectStringParser;
 import org.apache.zookeeper.client.HostProvider;
 import org.apache.zookeeper.client.StaticHostProvider;
-import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.ReflectionUtils;
@@ -32,7 +27,7 @@ import org.springframework.util.ReflectionUtils;
  * @author jianghang 2012-7-10 下午02:31:42
  * @version 1.0.0
  */
-public class ZooKeeperx implements IZkConnection {
+public class ZooKeeperx extends ZkConnection {
 
     private static final String SERVER_COMMA            = ";";
     private static final Logger logger                  = LoggerFactory.getLogger(ZooKeeperx.class);
@@ -40,12 +35,12 @@ public class ZooKeeperx implements IZkConnection {
     private static final Field  hostProviderField       = ReflectionUtils.findField(ClientCnxn.class, "hostProvider");
     private static final Field  serverAddressesField    = ReflectionUtils.findField(StaticHostProvider.class,
                                                             "serverAddresses");
+    private static final Field  zookeeperLockField      = ReflectionUtils.findField(ZkConnection.class,
+                                                            "_zookeeperLock");
+    private static final Field  zookeeperFiled          = ReflectionUtils.findField(ZkConnection.class, "_zk");
     private static final int    DEFAULT_SESSION_TIMEOUT = 90000;
 
-    private ZooKeeper           _zk                     = null;
-    private Lock                _zookeeperLock          = new ReentrantLock();
-
-    private final List<String>  _servers;
+    private final List<String>  _serversList;
     private final int           _sessionTimeOut;
 
     public ZooKeeperx(String zkServers){
@@ -53,114 +48,59 @@ public class ZooKeeperx implements IZkConnection {
     }
 
     public ZooKeeperx(String zkServers, int sessionTimeOut){
-        _servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA));
+        super(zkServers, sessionTimeOut);
+        _serversList = Arrays.asList(StringUtils.split(this.getServers(), SERVER_COMMA));
         _sessionTimeOut = sessionTimeOut;
     }
 
     @Override
     public void connect(Watcher watcher) {
+        ReflectionUtils.makeAccessible(zookeeperLockField);
+        ReflectionUtils.makeAccessible(zookeeperFiled);
+        Lock _zookeeperLock = (ReentrantLock) ReflectionUtils.getField(zookeeperLockField, this);
+        ZooKeeper _zk = (ZooKeeper) ReflectionUtils.getField(zookeeperFiled, this);
+
         _zookeeperLock.lock();
         try {
             if (_zk != null) {
                 throw new IllegalStateException("zk client has already been started");
             }
+            String zkServers = _serversList.get(0);
 
             try {
-                logger.debug("Creating new ZookKeeper instance to connect to " + _servers + ".");
-                _zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher);
+                logger.debug("Creating new ZookKeeper instance to connect to " + zkServers + ".");
+                _zk = new ZooKeeper(zkServers, _sessionTimeOut, watcher);
                 configMutliCluster(_zk);
+                ReflectionUtils.setField(zookeeperFiled, this, _zk);
             } catch (IOException e) {
-                throw new ZkException("Unable to connect to " + _servers, e);
+                throw new ZkException("Unable to connect to " + zkServers, e);
             }
         } finally {
             _zookeeperLock.unlock();
         }
     }
 
-    public void close() throws InterruptedException {
-        _zookeeperLock.lock();
-        try {
-            if (_zk != null) {
-                logger.debug("Closing ZooKeeper connected to " + _servers);
-                _zk.close();
-                _zk = null;
-            }
-        } finally {
-            _zookeeperLock.unlock();
-        }
-    }
-
-    public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException {
-        return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode);
-    }
-
-    public void delete(String path) throws InterruptedException, KeeperException {
-        _zk.delete(path, -1);
-    }
-
-    public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException {
-        return _zk.exists(path, watch) != null;
-    }
-
-    public List<String> getChildren(final String path, final boolean watch) throws KeeperException,
-                                                                           InterruptedException {
-        return _zk.getChildren(path, watch);
-    }
-
-    public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException {
-        return _zk.getData(path, watch, stat);
-    }
-
-    public void writeData(String path, byte[] data) throws KeeperException, InterruptedException {
-        writeData(path, data, -1);
-    }
-
-    public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException {
-        _zk.setData(path, data, version);
-    }
-
-    public States getZookeeperState() {
-        return _zk != null ? _zk.getState() : null;
-    }
-
-    public ZooKeeper getZookeeper() {
-        return _zk;
-    }
-
-    public long getCreateTime(String path) throws KeeperException, InterruptedException {
-        Stat stat = _zk.exists(path, false);
-        if (stat != null) {
-            return stat.getCtime();
-        }
-        return -1;
-    }
-
-    public String getServers() {
-        return StringUtils.join(_servers, SERVER_COMMA);
-    }
-
     // ===============================
 
     public void configMutliCluster(ZooKeeper zk) {
-        if (_servers.size() == 1) {
+        if (_serversList.size() == 1) {
             return;
         }
-        String cluster1 = _servers.get(0);
+        String cluster1 = _serversList.get(0);
         try {
-            if (_servers.size() > 1) {
+            if (_serversList.size() > 1) {
                 // 强制的声明accessible
                 ReflectionUtils.makeAccessible(clientCnxnField);
                 ReflectionUtils.makeAccessible(hostProviderField);
                 ReflectionUtils.makeAccessible(serverAddressesField);
 
                 // 添加第二组集群列表
-                for (int i = 1; i < _servers.size(); i++) {
-                    String cluster = _servers.get(i);
+                for (int i = 1; i < _serversList.size(); i++) {
+                    String cluster = _serversList.get(i);
                     // 强制获取zk中的地址信息
                     ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
                     HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
-                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(
-                                                                                                             serverAddressesField,
+                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(serverAddressesField,
                         hostProvider);
                     // 添加第二组集群列表
                     serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java

@@ -120,8 +120,10 @@ public class ZookeeperPathUtils {
     }
 
     public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) {
-        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName,
-                                    String.valueOf(clientId), getBatchMarkNode(batchId));
+        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH,
+            destinationName,
+            String.valueOf(clientId),
+            getBatchMarkNode(batchId));
     }
 
     public static String getCursorPath(String destination, short clientId) {

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java

@@ -87,6 +87,10 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
 
     }
 
+    public void init() {
+        processStart();
+    }
+
     public void start() {
         super.start();
         processStart();

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java

@@ -11,7 +11,8 @@ import java.util.Map;
 public class ServerRunningMonitors {
 
     private static ServerRunningData serverData;
-    private static Map               runningMonitors; // <String, ServerRunningMonitor>
+    private static Map               runningMonitors; // <String,
+                                                      // ServerRunningMonitor>
 
     public static ServerRunningData getServerData() {
         return serverData;

common/src/main/java/com/google/common/collect/MigrateMap.java

@@ -0,0 +1,19 @@
+package com.google.common.collect;
+
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.base.Function;
+
+public class MigrateMap {
+
+    @SuppressWarnings("deprecation")
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker,
+                                                              Function<? super K, ? extends V> computingFunction) {
+        return maker.makeComputingMap(computingFunction);
+    }
+
+    @SuppressWarnings("deprecation")
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
+        return new MapMaker().makeComputingMap(computingFunction);
+    }
+}

dbsync/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java

@@ -6,24 +6,24 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * An utility class implements MySQL/Java charsets conversion.  you can see com.mysql.jdbc.CharsetMapping.
+ * An utility class implements MySQL/Java charsets conversion. you can see
+ * com.mysql.jdbc.CharsetMapping.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  */
-public final class CharsetConversion
-{
+public final class CharsetConversion {
+
     static final Log logger = LogFactory.getLog(CharsetConversion.class);
 
-    static final class Entry
-    {
+    static final class Entry {
+
         protected final int    charsetId;
         protected final String mysqlCharset;
         protected final String mysqlCollation;
         protected final String javaCharset;
 
         Entry(final int id, String mysqlCharset, // NL
-                String mysqlCollation, String javaCharset)
-        {
+              String mysqlCollation, String javaCharset){
             this.charsetId = id;
             this.mysqlCharset = mysqlCharset;
             this.mysqlCollation = mysqlCollation;
@@ -34,38 +34,31 @@ public final class CharsetConversion
     // Character set data used in lookups. The array will be sparse.
     static final Entry[] entries = new Entry[0xff];
 
-    static Entry getEntry(final int id)
-    {
-        if (id >= 0 && id < entries.length)
-        {
+    static Entry getEntry(final int id) {
+        if (id >= 0 && id < entries.length) {
             return entries[id];
-        }
-        else
-        {
+        } else {
             throw new IllegalArgumentException("Invalid charset id: " + id);
         }
     }
 
     // Loads character set information.
-    static void putEntry(final int charsetId, String mysqlCharset,
-            String mysqlCollation, String javaCharset)
-    {
+    static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation, String javaCharset) {
         entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
-                mysqlCollation, javaCharset);
+            mysqlCollation,
+            javaCharset);
     }
 
     // Loads character set information.
     @Deprecated
-    static void putEntry(final int charsetId, String mysqlCharset,
-            String mysqlCollation)
-    {
+    static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation) {
         entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
-                mysqlCollation, /* Unknown java charset */null);
+            mysqlCollation, /* Unknown java charset */
+            null);
     }
 
     // Load character set data statically.
-    static
-    {
+    static {
         putEntry(1, "big5", "big5_chinese_ci", "Big5");
         putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
         putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
@@ -274,16 +267,12 @@ public final class CharsetConversion
     /**
      * Return defined charset name for mysql.
      */
-    public static String getCharset(final int id)
-    {
+    public static String getCharset(final int id) {
         Entry entry = getEntry(id);
 
-        if (entry != null)
-        {
+        if (entry != null) {
             return entry.mysqlCharset;
-        }
-        else
-        {
+        } else {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
@@ -292,16 +281,12 @@ public final class CharsetConversion
     /**
      * Return defined collaction name for mysql.
      */
-    public static String getCollation(final int id)
-    {
+    public static String getCollation(final int id) {
         Entry entry = getEntry(id);
 
-        if (entry != null)
-        {
+        if (entry != null) {
             return entry.mysqlCollation;
-        }
-        else
-        {
+        } else {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
@@ -310,55 +295,41 @@ public final class CharsetConversion
     /**
      * Return converted charset name for java.
      */
-    public static String getJavaCharset(final int id)
-    {
+    public static String getJavaCharset(final int id) {
         Entry entry = getEntry(id);
 
-        if (entry != null)
-        {
-            if (entry.javaCharset != null)
-            {
+        if (entry != null) {
+            if (entry.javaCharset != null) {
                 return entry.javaCharset;
-            }
-            else
-            {
-                logger.warn("Unknown java charset for: id = " + id
-                        + ", name = " + entry.mysqlCharset + ", coll = "
+            } else {
+                logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
                             + entry.mysqlCollation);
                 return null;
             }
-        }
-        else
-        {
+        } else {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
     }
 
-    public static void main(String[] args)
-    {
-        for (int i = 0; i < entries.length; i++)
-        {
+    public static void main(String[] args) {
+        for (int i = 0; i < entries.length; i++) {
             Entry entry = entries[i];
 
             System.out.print(i);
             System.out.print(',');
             System.out.print(' ');
-            if (entry != null)
-            {
+            if (entry != null) {
                 System.out.print(entry.mysqlCharset);
                 System.out.print(',');
                 System.out.print(' ');
                 System.out.print(entry.javaCharset);
-                if (entry.javaCharset != null)
-                {
+                if (entry.javaCharset != null) {
                     System.out.print(',');
                     System.out.print(' ');
                     System.out.print(Charset.forName(entry.javaCharset).name());
                 }
-            }
-            else
-            {
+            } else {
                 System.out.print("null");
             }
             System.out.println();

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java

@@ -21,16 +21,13 @@ import org.apache.commons.logging.LogFactory;
  * DirectLogFetcher fetcher = new DirectLogFetcher();
  * fetcher.open(conn, file, 0, 13);
  * 
- * while (fetcher.fetch())
- * {
+ * while (fetcher.fetch()) {
  *     LogEvent event;
- *     do
- *     {
+ *     do {
  *         event = decoder.decode(fetcher, context);
  * 
  *         // process log event.
- *     }
- *     while (event != null);
+ *     } while (event != null);
  * }
  * // connection closed.
  * </pre>
@@ -38,8 +35,8 @@ import org.apache.commons.logging.LogFactory;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DirectLogFetcher extends LogFetcher
-{
+public final class DirectLogFetcher extends LogFetcher {
+
     protected static final Log logger                          = LogFactory.getLog(DirectLogFetcher.class);
 
     /** Command to dump binlog */
@@ -64,71 +61,53 @@ public final class DirectLogFetcher extends LogFetcher
     private OutputStream       mysqlOutput;
     private InputStream        mysqlInput;
 
-    public DirectLogFetcher()
-    {
+    public DirectLogFetcher(){
         super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
     }
 
-    public DirectLogFetcher(final int initialCapacity)
-    {
+    public DirectLogFetcher(final int initialCapacity){
         super(initialCapacity, DEFAULT_GROWTH_FACTOR);
     }
 
-    public DirectLogFetcher(final int initialCapacity, final float growthFactor)
-    {
+    public DirectLogFetcher(final int initialCapacity, final float growthFactor){
         super(initialCapacity, growthFactor);
     }
 
-    private static final Object unwrapConnection(Object conn, Class<?> connClazz)
-            throws IOException
-    {
-        while (!connClazz.isInstance(conn))
-        {
-            try
-            {
+    private static final Object unwrapConnection(Object conn, Class<?> connClazz) throws IOException {
+        while (!connClazz.isInstance(conn)) {
+            try {
                 Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
-                if (connProxy.isInstance(conn))
-                {
+                if (connProxy.isInstance(conn)) {
                     conn = invokeMethod(conn, connProxy, "getTargetConnection");
                     continue;
                 }
-            }
-            catch (ClassNotFoundException e)
-            {
-                // org.springframework.jdbc.datasource.ConnectionProxy not found.
+            } catch (ClassNotFoundException e) {
+                // org.springframework.jdbc.datasource.ConnectionProxy not
+                // found.
             }
 
-            try
-            {
+            try {
                 Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
-                if (connProxy.isInstance(conn))
-                {
+                if (connProxy.isInstance(conn)) {
                     conn = getDeclaredField(conn, connProxy, "_conn");
                     continue;
                 }
-            }
-            catch (ClassNotFoundException e)
-            {
+            } catch (ClassNotFoundException e) {
                 // org.apache.commons.dbcp.DelegatingConnection not found.
             }
 
-            try
-            {
-                if (conn instanceof java.sql.Wrapper)
-                {
+            try {
+                if (conn instanceof java.sql.Wrapper) {
                     Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
                     conn = ((java.sql.Wrapper) conn).unwrap(connIface);
                     continue;
                 }
-            }
-            catch (ClassNotFoundException e)
-            {
+            } catch (ClassNotFoundException e) {
                 // com.mysql.jdbc.Connection not found.
-            }
-            catch (SQLException e)
-            {
-                logger.warn("Unwrap " + conn.getClass().getName() + " to "
-                        + connClazz.getName() + " failed: " + e.getMessage(), e);
+            } catch (SQLException e) {
+                logger.warn("Unwrap " + conn.getClass().getName() + " to " + connClazz.getName() + " failed: "
+                            + e.getMessage(),
+                    e);
             }
 
             return null;
@@ -136,127 +115,85 @@ public final class DirectLogFetcher extends LogFetcher
         return conn;
     }
 
-    private static final Object invokeMethod(Object obj, Class<?> objClazz,
-            String name)
-    {
-        try
-        {
+    private static final Object invokeMethod(Object obj, Class<?> objClazz, String name) {
+        try {
             Method method = objClazz.getMethod(name, (Class<?>[]) null);
             return method.invoke(obj, (Object[]) null);
-        }
-        catch (NoSuchMethodException e)
-        {
-            throw new IllegalArgumentException("No such method: \'" + name
-                    + "\' @ " + objClazz.getName(), e);
-        }
-        catch (IllegalAccessException e)
-        {
-            throw new IllegalArgumentException("Cannot invoke method: \'"
-                    + name + "\' @ " + objClazz.getName(), e);
-        }
-        catch (InvocationTargetException e)
-        {
-            throw new IllegalArgumentException("Invoke method failed: \'"
-                    + name + "\' @ " + objClazz.getName(),
+        } catch (NoSuchMethodException e) {
+            throw new IllegalArgumentException("No such method: \'" + name + "\' @ " + objClazz.getName(), e);
+        } catch (IllegalAccessException e) {
+            throw new IllegalArgumentException("Cannot invoke method: \'" + name + "\' @ " + objClazz.getName(), e);
+        } catch (InvocationTargetException e) {
+            throw new IllegalArgumentException("Invoke method failed: \'" + name + "\' @ " + objClazz.getName(),
                 e.getTargetException());
         }
     }
 
-    private static final Object getDeclaredField(Object obj, Class<?> objClazz,
-            String name)
-    {
-        try
-        {
+    private static final Object getDeclaredField(Object obj, Class<?> objClazz, String name) {
+        try {
             Field field = objClazz.getDeclaredField(name);
             field.setAccessible(true);
             return field.get(obj);
-        }
-        catch (NoSuchFieldException e)
-        {
-            throw new IllegalArgumentException("No such field: \'" + name
-                    + "\' @ " + objClazz.getName(), e);
-        }
-        catch (IllegalAccessException e)
-        {
-            throw new IllegalArgumentException("Cannot get field: \'" + name
-                    + "\' @ " + objClazz.getName(), e);
+        } catch (NoSuchFieldException e) {
+            throw new IllegalArgumentException("No such field: \'" + name + "\' @ " + objClazz.getName(), e);
+        } catch (IllegalAccessException e) {
+            throw new IllegalArgumentException("Cannot get field: \'" + name + "\' @ " + objClazz.getName(), e);
         }
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final int serverId)
-            throws IOException
-    {
+    public void open(Connection conn, String fileName, final int serverId) throws IOException {
         open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final int serverId,
-            boolean nonBlocking) throws IOException
-    {
+    public void open(Connection conn, String fileName, final int serverId, boolean nonBlocking) throws IOException {
         open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final long filePosition,
-            final int serverId) throws IOException
-    {
+    public void open(Connection conn, String fileName, final long filePosition, final int serverId) throws IOException {
         open(conn, fileName, filePosition, serverId, false);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, long filePosition,
-            final int serverId, boolean nonBlocking) throws IOException
-    {
-        try
-        {
+    public void open(Connection conn, String fileName, long filePosition, final int serverId, boolean nonBlocking)
+                                                                                                                  throws IOException {
+        try {
             this.conn = conn;
             Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
             Object unwrapConn = unwrapConnection(conn, connClazz);
-            if (unwrapConn == null)
-            {
-                throw new IOException("Unable to unwrap "
-                        + conn.getClass().getName()
+            if (unwrapConn == null) {
+                throw new IOException("Unable to unwrap " + conn.getClass().getName()
                                       + " to com.mysql.jdbc.ConnectionImpl");
             }
 
             // Get underlying IO streams for network communications.
             Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
-            if (connIo == null)
-            {
-                throw new IOException("Get null field:"
-                        + conn.getClass().getName() + "#io");
+            if (connIo == null) {
+                throw new IOException("Get null field:" + conn.getClass().getName() + "#io");
             }
-            mysqlOutput = (OutputStream) getDeclaredField(connIo,
-                    connIo.getClass(), "mysqlOutput");
-            mysqlInput = (InputStream) getDeclaredField(connIo,
-                    connIo.getClass(), "mysqlInput");
+            mysqlOutput = (OutputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlOutput");
+            mysqlInput = (InputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlInput");
 
-            if (filePosition == 0)
-                filePosition = BIN_LOG_HEADER_SIZE;
+            if (filePosition == 0) filePosition = BIN_LOG_HEADER_SIZE;
             sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
             position = 0;
-        }
-        catch (IOException e)
-        {
+        } catch (IOException e) {
             close(); /* Do cleanup */
-            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
-                    + ", position = " + filePosition);
+            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName + ", position = " + filePosition);
             throw e;
-        }
-        catch (ClassNotFoundException e)
-        {
+        } catch (ClassNotFoundException e) {
             close(); /* Do cleanup */
-            throw new IOException(
-                    "Unable to load com.mysql.jdbc.ConnectionImpl", e);
+            throw new IOException("Unable to load com.mysql.jdbc.ConnectionImpl", e);
         }
     }
 
@@ -265,8 +202,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param b the byte to put in the buffer
      */
-    protected final void putByte(byte b)
-    {
+    protected final void putByte(byte b) {
         ensureCapacity(position + 1);
 
         buffer[position++] = b;
@@ -277,8 +213,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param i16 the integer to put in the buffer
      */
-    protected final void putInt16(int i16)
-    {
+    protected final void putInt16(int i16) {
         ensureCapacity(position + 2);
 
         byte[] buf = buffer;
@@ -291,8 +226,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param i32 the integer to put in the buffer
      */
-    protected final void putInt32(long i32)
-    {
+    protected final void putInt32(long i32) {
         ensureCapacity(position + 4);
 
         byte[] buf = buffer;
@@ -307,8 +241,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param s the value to put in the buffer
      */
-    protected final void putString(String s)
-    {
+    protected final void putString(String s) {
         ensureCapacity(position + (s.length() * 2) + 1);
 
         System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
@@ -316,10 +249,8 @@ public final class DirectLogFetcher extends LogFetcher
         buffer[position++] = 0;
     }
 
-    protected final void sendBinlogDump(String fileName,
-            final long filePosition, final int serverId, boolean nonBlocking)
-            throws IOException
-    {
+    protected final void sendBinlogDump(String fileName, final long filePosition, final int serverId,
+                                        boolean nonBlocking) throws IOException {
         position = NET_HEADER_SIZE;
 
         putByte(COM_BINLOG_DUMP);
@@ -345,13 +276,10 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
      */
-    public boolean fetch() throws IOException
-    {
-        try
-        {
+    public boolean fetch() throws IOException {
+        try {
             // Fetching packet header from input.
-            if (!fetch0(0, NET_HEADER_SIZE))
-            {
+            if (!fetch0(0, NET_HEADER_SIZE)) {
                 logger.warn("Reached end of input stream while fetching header");
                 return false;
             }
@@ -359,61 +287,48 @@ public final class DirectLogFetcher extends LogFetcher
             // Fetching the first packet(may a multi-packet).
             int netlen = getUint24(PACKET_LEN_OFFSET);
             int netnum = getUint8(PACKET_SEQ_OFFSET);
-            if (!fetch0(NET_HEADER_SIZE, netlen))
-            {
-                logger.warn("Reached end of input stream: packet #" + netnum
-                        + ", len = " + netlen);
+            if (!fetch0(NET_HEADER_SIZE, netlen)) {
+                logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
                 return false;
             }
 
             // Detecting error code.
             final int mark = getUint8(NET_HEADER_SIZE);
-            if (mark != 0)
-            {
+            if (mark != 0) {
                 if (mark == 255) // error from master
                 {
-                    // Indicates an error, for example trying to fetch from wrong
+                    // Indicates an error, for example trying to fetch from
+                    // wrong
                     // binlog position.
                     position = NET_HEADER_SIZE + 1;
                     final int errno = getInt16();
                     String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
                     String errmsg = getFixString(limit - position);
-                    throw new IOException("Received error packet:"
-                            + " errno = " + errno + ", sqlstate = " + sqlstate
+                    throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
                                           + " errmsg = " + errmsg);
-                }
-                else if (mark == 254)
-                {
+                } else if (mark == 254) {
                     // Indicates end of stream. It's not clear when this would
                     // be sent.
-                    logger.warn("Received EOF packet from server, apparent"
-                            + " master disconnected.");
+                    logger.warn("Received EOF packet from server, apparent" + " master disconnected.");
                     return false;
-                }
-                else
-                {
+                } else {
                     // Should not happen.
-                    throw new IOException("Unexpected response " + mark
-                            + " while fetching binlog: packet #" + netnum
+                    throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
                                           + ", len = " + netlen);
                 }
             }
 
             // The first packet is a multi-packet, concatenate the packets.
-            while (netlen == MAX_PACKET_LENGTH)
-            {
-                if (!fetch0(0, NET_HEADER_SIZE))
-                {
+            while (netlen == MAX_PACKET_LENGTH) {
+                if (!fetch0(0, NET_HEADER_SIZE)) {
                     logger.warn("Reached end of input stream while fetching header");
                     return false;
                 }
 
                 netlen = getUint24(PACKET_LEN_OFFSET);
                 netnum = getUint8(PACKET_SEQ_OFFSET);
-                if (!fetch0(limit, netlen))
-                {
-                    logger.warn("Reached end of input stream: packet #"
-                            + netnum + ", len = " + netlen);
+                if (!fetch0(limit, netlen)) {
+                    logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
                     return false;
                 }
             }
@@ -423,43 +338,32 @@ public final class DirectLogFetcher extends LogFetcher
             position = origin;
             limit -= origin;
             return true;
-        }
-        catch (SocketTimeoutException e)
-        {
+        } catch (SocketTimeoutException e) {
             close(); /* Do cleanup */
             logger.error("Socket timeout expired, closing connection", e);
             throw e;
-        }
-        catch (InterruptedIOException e)
-        {
+        } catch (InterruptedIOException e) {
             close(); /* Do cleanup */
             logger.warn("I/O interrupted while reading from client socket", e);
             throw e;
-        }
-        catch (IOException e)
-        {
+        } catch (IOException e) {
             close(); /* Do cleanup */
             logger.error("I/O error while reading from client socket", e);
             throw e;
         }
     }
 
-    private final boolean fetch0(final int off, final int len)
-            throws IOException
-    {
+    private final boolean fetch0(final int off, final int len) throws IOException {
         ensureCapacity(off + len);
 
-        for (int count, n = 0; n < len; n += count)
-        {
-            if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
-            {
+        for (int count, n = 0; n < len; n += count) {
+            if (0 > (count = mysqlInput.read(buffer, off + n, len - n))) {
                 // Reached end of input stream
                 return false;
             }
         }
 
-        if (limit < off + len)
-            limit = off + len;
+        if (limit < off + len) limit = off + len;
         return true;
     }
 
@@ -468,19 +372,14 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
      */
-    public void close() throws IOException
-    {
-        try
-        {
-            if (conn != null)
-                conn.close();
+    public void close() throws IOException {
+        try {
+            if (conn != null) conn.close();
 
             conn = null;
             mysqlInput = null;
             mysqlOutput = null;
-        }
-        catch (SQLException e)
-        {
+        } catch (SQLException e) {
             logger.warn("Unable to close connection", e);
         }
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java

@@ -129,11 +129,11 @@ public final class FileLogFetcher extends LogFetcher {
             }
         } else if (limit > 0) {
             System.arraycopy(buffer, origin, buffer, 0, limit);
+            position -= origin;
+            origin = 0;
             final int len = fin.read(buffer, limit, buffer.length - limit);
             if (len >= 0) {
                 limit += len;
-                position -= origin;
-                origin = 0;
 
                 /* More binlog to fetch */
                 return true;

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java

@@ -0,0 +1,418 @@
+package com.taobao.tddl.dbsync.binlog;
+
+/**
+ * 处理下MySQL json二进制转化为可读的字符串
+ * 
+ * @author agapple 2016年6月30日 上午11:26:17
+ * @since 1.0.22
+ */
+public class JsonConversion {
+
+    // JSON TYPE
+    public static final int  JSONB_TYPE_SMALL_OBJECT = 0x0;
+    public static final int  JSONB_TYPE_LARGE_OBJECT = 0x1;
+    public static final int  JSONB_TYPE_SMALL_ARRAY  = 0x2;
+    public static final int  JSONB_TYPE_LARGE_ARRAY  = 0x3;
+    public static final int  JSONB_TYPE_LITERAL      = 0x4;
+    public static final int  JSONB_TYPE_INT16        = 0x5;
+    public static final int  JSONB_TYPE_UINT16       = 0x6;
+    public static final int  JSONB_TYPE_INT32        = 0x7;
+    public static final int  JSONB_TYPE_UINT32       = 0x8;
+    public static final int  JSONB_TYPE_INT64        = 0x9;
+    public static final int  JSONB_TYPE_UINT64       = 0xA;
+    public static final int  JSONB_TYPE_DOUBLE       = 0xB;
+    public static final int  JSONB_TYPE_STRING       = 0xC;
+    public static final int  JSONB_TYPE_OPAQUE       = 0xF;
+    public static final char JSONB_NULL_LITERAL      = '\0';
+    public static final char JSONB_TRUE_LITERAL      = '\1';
+    public static final char JSONB_FALSE_LITERAL     = '\2';
+
+    /*
+     * The size of offset or size fields in the small and the large storage
+     * format for JSON objects and JSON arrays.
+     */
+    public static final int  SMALL_OFFSET_SIZE       = 2;
+    public static final int  LARGE_OFFSET_SIZE       = 4;
+
+    /*
+     * The size of key entries for objects when using the small storage format
+     * or the large storage format. In the small format it is 4 bytes (2 bytes
+     * for key length and 2 bytes for key offset). In the large format it is 6
+     * (2 bytes for length, 4 bytes for offset).
+     */
+    public static final int  KEY_ENTRY_SIZE_SMALL    = (2 + SMALL_OFFSET_SIZE);
+    public static final int  KEY_ENTRY_SIZE_LARGE    = (2 + LARGE_OFFSET_SIZE);
+
+    /*
+     * The size of value entries for objects or arrays. When using the small
+     * storage format, the entry size is 3 (1 byte for type, 2 bytes for
+     * offset). When using the large storage format, it is 5 (1 byte for type, 4
+     * bytes for offset).
+     */
+    public static final int  VALUE_ENTRY_SIZE_SMALL  = (1 + SMALL_OFFSET_SIZE);
+    public static final int  VALUE_ENTRY_SIZE_LARGE  = (1 + LARGE_OFFSET_SIZE);
+
+    public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
+        buffer = buffer.duplicate(buffer.position(), (int) len);
+        switch (type) {
+            case JSONB_TYPE_SMALL_OBJECT:
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
+            case JSONB_TYPE_LARGE_OBJECT:
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
+            case JSONB_TYPE_SMALL_ARRAY:
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
+            case JSONB_TYPE_LARGE_ARRAY:
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
+            default:
+                return parse_scalar(type, buffer, len, charsetName);
+        }
+    }
+
+    private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
+                                                    String charsetName) {
+        long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+        if (len < 2 * offset_size) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        long element_count = read_offset_or_size(buffer, large);
+        long bytes = read_offset_or_size(buffer, large);
+
+        if (bytes > len) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        long header_size = 2 * offset_size;
+        if (type == Json_enum_type.OBJECT) {
+            header_size += element_count * (large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL);
+        }
+
+        header_size += element_count * (large ? VALUE_ENTRY_SIZE_LARGE : VALUE_ENTRY_SIZE_SMALL);
+        if (header_size > bytes) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        return new Json_Value(type, buffer.rewind(), element_count, bytes, large);
+    }
+
+    private static long read_offset_or_size(LogBuffer buffer, boolean large) {
+        return large ? buffer.getUint32() : buffer.getUint16();
+    }
+
+    private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
+        switch (type) {
+            case JSONB_TYPE_LITERAL:
+                /* purecov: inspected */
+                int data = buffer.getUint8();
+                switch (data) {
+                    case JSONB_NULL_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_NULL);
+                    case JSONB_TRUE_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_TRUE);
+                    case JSONB_FALSE_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_FALSE);
+                    default:
+                        throw new IllegalArgumentException("illegal json data");
+                }
+            case JSONB_TYPE_INT16:
+                return new Json_Value(Json_enum_type.INT, buffer.getInt16());
+            case JSONB_TYPE_INT32:
+                return new Json_Value(Json_enum_type.INT, buffer.getInt32());
+            case JSONB_TYPE_INT64:
+                return new Json_Value(Json_enum_type.INT, buffer.getLong64());
+            case JSONB_TYPE_UINT16:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUint16());
+            case JSONB_TYPE_UINT32:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUint32());
+            case JSONB_TYPE_UINT64:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUlong64());
+            case JSONB_TYPE_DOUBLE:
+                return new Json_Value(Json_enum_type.DOUBLE, Double.valueOf(buffer.getDouble64()));
+            case JSONB_TYPE_STRING:
+                int max_bytes = (int) Math.min(len, 5);
+                long tlen = 0;
+                long str_len = 0;
+                long n = 0;
+                byte[] datas = buffer.getData(max_bytes);
+                for (int i = 0; i < max_bytes; i++) {
+                    // Get the next 7 bits of the length.
+                    tlen |= (datas[i] & 0x7f) << (7 * i);
+                    if ((datas[i] & 0x80) == 0) {
+                        // The length shouldn't exceed 32 bits.
+                        if (tlen > 4294967296L) {
+                            throw new IllegalArgumentException("illegal json data");
+                        }
+
+                        // This was the last byte. Return successfully.
+                        n = i + 1;
+                        str_len = tlen;
+                        break;
+                    }
+                }
+
+                if (len < n + str_len) {
+                    throw new IllegalArgumentException("illegal json data");
+                }
+                return new Json_Value(Json_enum_type.STRING, buffer.rewind()
+                    .forward((int) n)
+                    .getFixString((int) str_len, charsetName));
+            case JSONB_TYPE_OPAQUE:
+                /*
+                 * There should always be at least one byte, which tells the
+                 * field type of the opaque value.
+                 */
+                // The type is encoded as a uint8 that maps to an
+                // enum_field_types.
+                int type_byte = buffer.getUint8();
+                int position = buffer.position();
+                // Then there's the length of the value.
+                int q_max_bytes = (int) Math.min(len, 5);
+                long q_tlen = 0;
+                long q_str_len = 0;
+                long q_n = 0;
+                byte[] q_datas = buffer.getData(q_max_bytes);
+                for (int i = 0; i < q_max_bytes; i++) {
+                    // Get the next 7 bits of the length.
+                    q_tlen |= (q_datas[i] & 0x7f) << (7 * i);
+                    if ((q_datas[i] & 0x80) == 0) {
+                        // The length shouldn't exceed 32 bits.
+                        if (q_tlen > 4294967296L) {
+                            throw new IllegalArgumentException("illegal json data");
+                        }
+
+                        // This was the last byte. Return successfully.
+                        q_n = i + 1;
+                        q_str_len = q_tlen;
+                        break;
+                    }
+                }
+
+                if (q_str_len == 0 || len < q_n + q_str_len) {
+                    throw new IllegalArgumentException("illegal json data");
+                }
+                return new Json_Value(type_byte, buffer.position(position).forward((int) q_n), q_str_len);
+            default:
+                throw new IllegalArgumentException("illegal json data");
+        }
+    }
+
+    public static class Json_Value {
+
+        Json_enum_type m_type;
+        int            m_field_type;
+        LogBuffer      m_data;
+        long           m_element_count;
+        long           m_length;
+        String         m_string_value;
+        Number         m_int_value;
+        double         m_double_value;
+        boolean        m_large;
+
+        public Json_Value(Json_enum_type t){
+            this.m_type = t;
+        }
+
+        public Json_Value(Json_enum_type t, Number val){
+            this.m_type = t;
+            if (t == Json_enum_type.DOUBLE) {
+                this.m_double_value = val.doubleValue();
+            } else {
+                this.m_int_value = val;
+            }
+        }
+
+        public Json_Value(Json_enum_type t, String value){
+            this.m_type = t;
+            this.m_string_value = value;
+        }
+
+        public Json_Value(int field_type, LogBuffer data, long bytes){
+            this.m_type = Json_enum_type.OPAQUE; // 不确定类型
+            this.m_field_type = field_type;
+            this.m_data = data;
+            this.m_length = bytes;
+        }
+
+        public Json_Value(Json_enum_type t, LogBuffer data, long element_count, long bytes, boolean large){
+            this.m_type = t;
+            this.m_data = data;
+            this.m_element_count = element_count;
+            this.m_length = bytes;
+            this.m_large = large;
+        }
+
+        public String key(int i, String charsetName) {
+            m_data.rewind();
+            int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+            int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
+            int entry_offset = 2 * offset_size + key_entry_size * i;
+            // The offset of the key is the first part of the key
+            // entry.
+            m_data.forward(entry_offset);
+            long key_offset = read_offset_or_size(m_data, m_large);
+            // The length of the key is the second part of the
+            // entry, always two
+            // bytes.
+            long key_length = m_data.getUint16();
+            return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
+        }
+
+        public Json_Value element(int i, String charsetName) {
+            m_data.rewind();
+            int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+            int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
+            int value_entry_size = m_large ? VALUE_ENTRY_SIZE_LARGE : VALUE_ENTRY_SIZE_SMALL;
+            int first_entry_offset = 2 * offset_size;
+            if (m_type == Json_enum_type.OBJECT) {
+                first_entry_offset += m_element_count * key_entry_size;
+            }
+            int entry_offset = first_entry_offset + value_entry_size * i;
+            int type = m_data.forward(entry_offset).getUint8();
+            if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
+                || (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
+                return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
+            }
+            int value_offset = (int) read_offset_or_size(m_data, m_large);
+            return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
+        }
+
+        public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
+            switch (m_type) {
+                case OBJECT:
+                    buf.append("{");
+                    for (int i = 0; i < m_element_count; ++i) {
+                        if (i > 0) {
+                            buf.append(", ");
+                        }
+                        buf.append('"').append(key(i, charsetName)).append('"');
+                        buf.append(": ");
+                        element(i, charsetName).toJsonString(buf, charsetName);
+                    }
+                    buf.append("}");
+                    break;
+                case ARRAY:
+                    buf.append("[");
+                    for (int i = 0; i < m_element_count; ++i) {
+                        if (i > 0) {
+                            buf.append(", ");
+                        }
+                        element(i, charsetName).toJsonString(buf, charsetName);
+                    }
+                    buf.append("]");
+                    break;
+                case DOUBLE:
+                    buf.append(Double.valueOf(m_double_value).toString());
+                    break;
+                case INT:
+                    buf.append(m_int_value.toString());
+                    break;
+                case UINT:
+                    buf.append(m_int_value.toString());
+                    break;
+                case LITERAL_FALSE:
+                    buf.append("false");
+                    break;
+                case LITERAL_TRUE:
+                    buf.append("true");
+                    break;
+                case LITERAL_NULL:
+                    buf.append("NULL");
+                    break;
+                case OPAQUE:
+                    String text = null;
+                    if (m_field_type == LogEvent.MYSQL_TYPE_NEWDECIMAL) {
+                        int precision = m_data.getInt8();
+                        int scale = m_data.getInt8();
+                        text = m_data.getDecimal(precision, scale).toPlainString();
+                        buf.append(text);
+                    } else if (m_field_type == LogEvent.MYSQL_TYPE_TIME) {
+                        long packed_value = m_data.getLong64();
+                        if (packed_value == 0) {
+                            text = "00:00:00";
+                        } else {
+                            long ultime = Math.abs(packed_value);
+                            long intpart = ultime >> 24;
+                            int frac = (int) (ultime % (1L << 24));
+                            text = String.format("%s%02d:%02d:%02d",
+                                packed_value >= 0 ? "" : "-",
+                                (int) ((intpart >> 12) % (1 << 10)),
+                                (int) ((intpart >> 6) % (1 << 6)),
+                                (int) (intpart % (1 << 6)));
+                            text = text + "." + usecondsToStr(frac, 6);
+                        }
+                        buf.append('"').append(text).append('"');
+                    } else if (m_field_type == LogEvent.MYSQL_TYPE_DATE || m_field_type == LogEvent.MYSQL_TYPE_DATETIME
+                               || m_field_type == LogEvent.MYSQL_TYPE_TIMESTAMP) {
+                        long packed_value = m_data.getLong64();
+                        if (packed_value == 0) {
+                            text = "0000-00-00 00:00:00";
+                        } else {
+                            // 构造TimeStamp只处理到秒
+                            long ultime = Math.abs(packed_value);
+                            long intpart = ultime >> 24;
+                            int frac = (int) (ultime % (1L << 24));
+                            long ymd = intpart >> 17;
+                            long ym = ymd >> 5;
+                            long hms = intpart % (1 << 17);
+                            text = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                                (int) (ym / 13),
+                                (int) (ym % 13),
+                                (int) (ymd % (1 << 5)),
+                                (int) (hms >> 12),
+                                (int) ((hms >> 6) % (1 << 6)),
+                                (int) (hms % (1 << 6)));
+                            text = text + "." + usecondsToStr(frac, 6);
+                        }
+                        buf.append('"').append(text).append('"');
+                    } else {
+                        text = m_data.getFixString((int) m_length, charsetName);
+                        buf.append('"').append(escapse(text)).append('"');
+                    }
+
+                    break;
+                case STRING:
+                    buf.append('"').append(escapse(m_string_value)).append('"');
+                    break;
+                case ERROR:
+                    throw new IllegalArgumentException("illegal json data");
+            }
+
+            return buf;
+        }
+    }
+
+    private static StringBuilder escapse(String data) {
+        StringBuilder sb = new StringBuilder(data.length());
+        int endIndex = data.length();
+        for (int i = 0; i < endIndex; ++i) {
+            char c = data.charAt(i);
+            if (c == '"') {
+                sb.append('\\');
+            } else if (c == '\\') {
+                sb.append("\\");
+            }
+            sb.append(c);
+        }
+        return sb;
+    }
+
+    public static enum Json_enum_type {
+        OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR
+    }
+
+    private static String usecondsToStr(int frac, int meta) {
+        String sec = String.valueOf(frac);
+        if (meta > 6) {
+            throw new IllegalArgumentException("unknow useconds meta : " + meta);
+        }
+
+        if (sec.length() < 6) {
+            StringBuilder result = new StringBuilder(6);
+            int len = 6 - sec.length();
+            for (; len > 0; len--) {
+                result.append('0');
+            }
+            result.append(sec);
+            sec = result.toString();
+        }
+
+        return sec.substring(0, meta);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java

@@ -7,69 +7,56 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
 
 /**
- * TODO: Document Me!!
- * 
- * NOTE: Log context will NOT write multi-threaded.
+ * TODO: Document Me!! NOTE: Log context will NOT write multi-threaded.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogContext
-{
+public final class LogContext {
+
     private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
 
     private FormatDescriptionLogEvent         formatDescription;
 
     private LogPosition                       logPosition;
 
-    public LogContext()
-    {
+    public LogContext(){
         this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
     }
 
-    public LogContext(FormatDescriptionLogEvent descriptionEvent)
-    {
+    public LogContext(FormatDescriptionLogEvent descriptionEvent){
         this.formatDescription = descriptionEvent;
     }
 
-    public final LogPosition getLogPosition()
-    {
+    public final LogPosition getLogPosition() {
         return logPosition;
     }
 
-    public final void setLogPosition(LogPosition logPosition)
-    {
+    public final void setLogPosition(LogPosition logPosition) {
         this.logPosition = logPosition;
     }
 
-    public final FormatDescriptionLogEvent getFormatDescription()
-    {
+    public final FormatDescriptionLogEvent getFormatDescription() {
         return formatDescription;
     }
 
-    public final void setFormatDescription(
-            FormatDescriptionLogEvent formatDescription)
-    {
+    public final void setFormatDescription(FormatDescriptionLogEvent formatDescription) {
         this.formatDescription = formatDescription;
     }
 
-    public final void putTable(TableMapLogEvent mapEvent)
-    {
+    public final void putTable(TableMapLogEvent mapEvent) {
         mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
     }
 
-    public final TableMapLogEvent getTable(final long tableId)
-    {
+    public final TableMapLogEvent getTable(final long tableId) {
         return mapOfTable.get(Long.valueOf(tableId));
     }
 
-    public final void clearAllTables()
-    {
+    public final void clearAllTables() {
         mapOfTable.clear();
     }
 
-    public void reset()
-    {
+    public void reset() {
         formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
 
         mapOfTable.clear();

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -61,28 +61,24 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogDecoder
-{
+public final class LogDecoder {
+
     protected static final Log logger    = LogFactory.getLog(LogDecoder.class);
 
     protected final BitSet     handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
 
-    public LogDecoder()
-    {
+    public LogDecoder(){
     }
 
-    public LogDecoder(final int fromIndex, final int toIndex)
-    {
+    public LogDecoder(final int fromIndex, final int toIndex){
         handleSet.set(fromIndex, toIndex);
     }
 
-    public final void handle(final int fromIndex, final int toIndex)
-    {
+    public final void handle(final int fromIndex, final int toIndex) {
         handleSet.set(fromIndex, toIndex);
     }
 
-    public final void handle(final int flagIndex)
-    {
+    public final void handle(final int flagIndex) {
         handleSet.set(flagIndex);
     }
 
@@ -92,46 +88,30 @@ public final class LogDecoder
      * @return <code>UknownLogEvent</code> if event type is unknown or skipped,
      * <code>null</code> if buffer is not including a full event.
      */
-    public LogEvent decode(LogBuffer buffer, LogContext context)
-            throws IOException
-    {
+    public LogEvent decode(LogBuffer buffer, LogContext context) throws IOException {
         final int limit = buffer.limit();
 
-        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
-        {
-            LogHeader header = new LogHeader(buffer,
-                    context.getFormatDescription());
+        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
+            LogHeader header = new LogHeader(buffer, context.getFormatDescription());
 
             final int len = header.getEventLen();
-            if (limit >= len)
-            {
+            if (limit >= len) {
                 LogEvent event;
 
                 /* Checking binary-log's header */
-                if (handleSet.get(header.getType()))
-                {
+                if (handleSet.get(header.getType())) {
                     buffer.limit(len);
-                    try
-                    {
+                    try {
                         /* Decoding binary-log to event */
                         event = decode(buffer, header, context);
-                    }
-                    catch (IOException e)
-                    {
-                        if (logger.isWarnEnabled())
-                            logger.warn("Decoding "
-                                    + LogEvent.getTypeName(header.getType())
-                                    + " failed from: "
-                                    + context.getLogPosition(), e);
+                    } catch (IOException e) {
+                        if (logger.isWarnEnabled()) logger.warn("Decoding " + LogEvent.getTypeName(header.getType())
+                                                                + " failed from: " + context.getLogPosition(), e);
                         throw e;
-                    }
-                    finally
-                    {
+                    } finally {
                         buffer.limit(limit); /* Restore limit */
                     }
-                }
-                else
-                {
+                } else {
                     /* Ignore unsupported binary-log. */
                     event = new UnknownLogEvent(header);
                 }
@@ -152,9 +132,7 @@ public final class LogDecoder
      * 
      * @return <code>UknownLogEvent</code> if event type is unknown or skipped.
      */
-    public static LogEvent decode(LogBuffer buffer, LogHeader header,
-            LogContext context) throws IOException
-    {
+    public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext context) throws IOException {
         FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
         LogPosition logPosition = context.getLogPosition();
 
@@ -171,234 +149,179 @@ public final class LogDecoder
             buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
         }
 
-        switch (header.getType())
-        {
-        case LogEvent.QUERY_EVENT:
-            {
-                QueryLogEvent event = new QueryLogEvent(header, buffer,
-                        descriptionEvent);
+        switch (header.getType()) {
+            case LogEvent.QUERY_EVENT: {
+                QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.XID_EVENT:
-            {
-                XidLogEvent event = new XidLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.XID_EVENT: {
+                XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.TABLE_MAP_EVENT:
-            {
-                TableMapLogEvent mapEvent = new TableMapLogEvent(header,
-                        buffer, descriptionEvent);
+            case LogEvent.TABLE_MAP_EVENT: {
+                TableMapLogEvent mapEvent = new TableMapLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 context.putTable(mapEvent);
                 return mapEvent;
             }
-        case LogEvent.WRITE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.WRITE_ROWS_EVENT_V1: {
+                RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.UPDATE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.UPDATE_ROWS_EVENT_V1: {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.DELETE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.DELETE_ROWS_EVENT_V1: {
+                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.ROTATE_EVENT:
-            {
-                RotateLogEvent event = new RotateLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.ROTATE_EVENT: {
+                RotateLogEvent event = new RotateLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
-                logPosition = new LogPosition(event.getFilename(),
-                        event.getPosition());
+                logPosition = new LogPosition(event.getFilename(), event.getPosition());
                 context.setLogPosition(logPosition);
                 return event;
             }
             case LogEvent.LOAD_EVENT:
-        case LogEvent.NEW_LOAD_EVENT:
-            {
-                LoadLogEvent event = new LoadLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.NEW_LOAD_EVENT: {
+                LoadLogEvent event = new LoadLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
             case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
             {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported SLAVE_EVENT from: "
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: "
                                                         + context.getLogPosition());
                 break;
             }
-        case LogEvent.CREATE_FILE_EVENT:
-            {
-                CreateFileLogEvent event = new CreateFileLogEvent(header,
-                        buffer, descriptionEvent);
+            case LogEvent.CREATE_FILE_EVENT: {
+                CreateFileLogEvent event = new CreateFileLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.APPEND_BLOCK_EVENT:
-            {
-                AppendBlockLogEvent event = new AppendBlockLogEvent(header,
-                        buffer, descriptionEvent);
+            case LogEvent.APPEND_BLOCK_EVENT: {
+                AppendBlockLogEvent event = new AppendBlockLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.DELETE_FILE_EVENT:
-            {
-                DeleteFileLogEvent event = new DeleteFileLogEvent(header,
-                        buffer, descriptionEvent);
+            case LogEvent.DELETE_FILE_EVENT: {
+                DeleteFileLogEvent event = new DeleteFileLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.EXEC_LOAD_EVENT:
-            {
-                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
-                        buffer, descriptionEvent);
+            case LogEvent.EXEC_LOAD_EVENT: {
+                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.START_EVENT_V3:
-            {
+            case LogEvent.START_EVENT_V3: {
                 /* This is sent only by MySQL <=4.x */
-                StartLogEventV3 event = new StartLogEventV3(header, buffer,
-                        descriptionEvent);
+                StartLogEventV3 event = new StartLogEventV3(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.STOP_EVENT:
-            {
-                StopLogEvent event = new StopLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.STOP_EVENT: {
+                StopLogEvent event = new StopLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.INTVAR_EVENT:
-            {
-                IntvarLogEvent event = new IntvarLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.INTVAR_EVENT: {
+                IntvarLogEvent event = new IntvarLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.RAND_EVENT:
-            {
-                RandLogEvent event = new RandLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.RAND_EVENT: {
+                RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.USER_VAR_EVENT:
-            {
-                UserVarLogEvent event = new UserVarLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.USER_VAR_EVENT: {
+                UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.FORMAT_DESCRIPTION_EVENT:
-            {
-                descriptionEvent = new FormatDescriptionLogEvent(header,
-                        buffer, descriptionEvent);
+            case LogEvent.FORMAT_DESCRIPTION_EVENT: {
+                descriptionEvent = new FormatDescriptionLogEvent(header, buffer, descriptionEvent);
                 context.setFormatDescription(descriptionEvent);
                 return descriptionEvent;
             }
-        case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+            case LogEvent.PRE_GA_WRITE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
                                                         + context.getLogPosition());
                 // ev = new Write_rows_log_event_old(buf, event_len,
                 // description_event);
                 break;
             }
-        case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+            case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
                                                         + context.getLogPosition());
                 // ev = new Update_rows_log_event_old(buf, event_len,
                 // description_event);
                 break;
             }
-        case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+            case LogEvent.PRE_GA_DELETE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
                                                         + context.getLogPosition());
                 // ev = new Delete_rows_log_event_old(buf, event_len,
                 // description_event);
                 break;
             }
-        case LogEvent.BEGIN_LOAD_QUERY_EVENT:
-            {
-                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
-                        header, buffer, descriptionEvent);
+            case LogEvent.BEGIN_LOAD_QUERY_EVENT: {
+                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
-            {
-                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
-                        header, buffer, descriptionEvent);
+            case LogEvent.EXECUTE_LOAD_QUERY_EVENT: {
+                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.INCIDENT_EVENT:
-            {
-                IncidentLogEvent event = new IncidentLogEvent(header, buffer,
-                        descriptionEvent);
+            case LogEvent.INCIDENT_EVENT: {
+                IncidentLogEvent event = new IncidentLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.HEARTBEAT_LOG_EVENT:
-        {
-            HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
-                    descriptionEvent);
+            case LogEvent.HEARTBEAT_LOG_EVENT: {
+                HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.IGNORABLE_LOG_EVENT:
-            {
+            case LogEvent.IGNORABLE_LOG_EVENT: {
                 IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.ROWS_QUERY_LOG_EVENT:
-            {
+            case LogEvent.ROWS_QUERY_LOG_EVENT: {
                 RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
@@ -426,43 +349,37 @@ public final class LogDecoder
                 return event;
             }
             case LogEvent.GTID_LOG_EVENT:
-        case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
-            {
+            case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
                 GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
-            {
+            case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
                 PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.ANNOTATE_ROWS_EVENT:
-            {
+            case LogEvent.ANNOTATE_ROWS_EVENT: {
                 AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.BINLOG_CHECKPOINT_EVENT:
-            {
+            case LogEvent.BINLOG_CHECKPOINT_EVENT: {
                 BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.GTID_EVENT:
-            {
+            case LogEvent.GTID_EVENT: {
                 MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.GTID_LIST_EVENT:
-            {
+            case LogEvent.GTID_LIST_EVENT: {
                 MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
@@ -470,8 +387,9 @@ public final class LogDecoder
             }
             default:
                 /*
-                Create an object of Ignorable_log_event for unrecognized sub-class.
-                So that SLAVE SQL THREAD will only update the position and continue.
+                 * Create an object of Ignorable_log_event for unrecognized
+                 * sub-class. So that SLAVE SQL THREAD will only update the
+                 * position and continue.
                  */
                 if ((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0) {
                     IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
@@ -479,8 +397,7 @@ public final class LogDecoder
                     logPosition.position = header.getLogPos();
                     return event;
                 } else {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unrecognized binlog event "
+                    if (logger.isWarnEnabled()) logger.warn("Skipping unrecognized binlog event "
                                                             + LogEvent.getTypeName(header.getType()) + " from: "
                                                             + context.getLogPosition());
                 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java

@@ -7,88 +7,66 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
 
 /**
  * Binary log event definitions. This includes generic code common to all types
- * of log events, as well as specific code for each type of log event.
- * 
- * - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
- * in little endian, i.e., the least significant byte first, unless otherwise
- * specified.
- * 
- * representation of unsigned integers, called Packed Integer. A Packed Integer
- * has the capacity of storing up to 8-byte integers, while small integers still
- * can use 1, 3, or 4 bytes. The value of the first byte determines how to read
- * the number, according to the following table:
- * 
+ * of log events, as well as specific code for each type of log event. - All
+ * numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored in
+ * little endian, i.e., the least significant byte first, unless otherwise
+ * specified. representation of unsigned integers, called Packed Integer. A
+ * Packed Integer has the capacity of storing up to 8-byte integers, while small
+ * integers still can use 1, 3, or 4 bytes. The value of the first byte
+ * determines how to read the number, according to the following table:
  * <table>
  * <caption>Format of Packed Integer</caption>
- * 
  * <tr>
  * <th>First byte</th>
  * <th>Format</th>
  * </tr>
- * 
  * <tr>
  * <td>0-250</td>
  * <td>The first byte is the number (in the range 0-250), and no more bytes are
  * used.</td>
  * </tr>
- * 
  * <tr>
  * <td>252</td>
  * <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
  * </tr>
- * 
  * <tr>
  * <td>253</td>
  * <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
  * </tr>
- * 
  * <tr>
  * <td>254</td>
  * <td>Eight more bytes are used. The number is in the range
  * 0xffffff-0xffffffffffffffff.</td>
  * </tr>
- * 
  * </table>
- * 
  * - Strings are stored in various formats. The format of each string is
  * documented separately.
  * 
  * @see mysql-5.1.60/sql/log_event.h
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class LogEvent
-{
+public abstract class LogEvent {
+
     /*
-     * 3 is MySQL 4.x; 4 is MySQL 5.0.0.
-     * Compared to version 3, version 4 has:
-     * - a different Start_log_event, which includes info about the binary log
+     * 3 is MySQL 4.x; 4 is MySQL 5.0.0. Compared to version 3, version 4 has: -
+     * a different Start_log_event, which includes info about the binary log
      * (sizes of headers); this info is included for better compatibility if the
-     * master's MySQL version is different from the slave's.
-     * - all events have a unique ID (the triplet (server_id, timestamp at server
-     * start, other) to be sure an event is not executed more than once in a
-     * multimaster setup, example:
-     *              M1
-     *            /   \
-     *           v     v
-     *           M2    M3
-     *           \     /
-     *            v   v
-     *              S
-     * if a query is run on M1, it will arrive twice on S, so we need that S
-     * remembers the last unique ID it has processed, to compare and know if the
-     * event should be skipped or not. Example of ID: we already have the server id
-     * (4 bytes), plus:
+     * master's MySQL version is different from the slave's. - all events have a
+     * unique ID (the triplet (server_id, timestamp at server start, other) to
+     * be sure an event is not executed more than once in a multimaster setup,
+     * example: M1 / \ v v M2 M3 \ / v v S if a query is run on M1, it will
+     * arrive twice on S, so we need that S remembers the last unique ID it has
+     * processed, to compare and know if the event should be skipped or not.
+     * Example of ID: we already have the server id (4 bytes), plus:
      * timestamp_when_the_master_started (4 bytes), a counter (a sequence number
      * which increments every time we write an event to the binlog) (3 bytes).
      * Q: how do we handle when the counter is overflowed and restarts from 0 ?
-     * 
      * - Query and Load (Create or Execute) events may have a more precise
      * timestamp (with microseconds), number of matched/affected/warnings rows
-     * and fields of session variables: SQL_MODE,
-     * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
-     * charsets, the PASSWORD() version (old/new/...).
+     * and fields of session variables: SQL_MODE, FOREIGN_KEY_CHECKS,
+     * UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and charsets, the
+     * PASSWORD() version (old/new/...).
      */
     public static final int    BINLOG_VERSION                           = 4;
 
@@ -160,13 +138,15 @@ public abstract class LogEvent
     public static final int    INCIDENT_EVENT                           = 26;
 
     /**
-     * Heartbeat event to be send by master at its idle time to ensure master's online status to slave
+     * Heartbeat event to be send by master at its idle time to ensure master's
+     * online status to slave
      */
     public static final int    HEARTBEAT_LOG_EVENT                      = 27;
 
     /**
-     * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
-     * case there is code for handling it, but which can be ignored if it is not recognized.
+     * In some situations, it is necessary to send over ignorable data to the
+     * slave: data that a slave can handle in case there is code for handling
+     * it, but which can be ignored if it is not recognized.
      */
     public static final int    IGNORABLE_LOG_EVENT                      = 28;
     public static final int    ROWS_QUERY_LOG_EVENT                     = 29;
@@ -189,23 +169,23 @@ public abstract class LogEvent
     /* New Maria event numbers start from here */
     public static final int    ANNOTATE_ROWS_EVENT                      = 160;
     /*
-    Binlog checkpoint event. Used for XA crash recovery on the master, not used
-    in replication.
-    A binlog checkpoint event specifies a binlog file such that XA crash
-    recovery can start from that file - and it is guaranteed to find all XIDs
-    that are prepared in storage engines but not yet committed.
+     * Binlog checkpoint event. Used for XA crash recovery on the master, not
+     * used in replication. A binlog checkpoint event specifies a binlog file
+     * such that XA crash recovery can start from that file - and it is
+     * guaranteed to find all XIDs that are prepared in storage engines but not
+     * yet committed.
      */
     public static final int    BINLOG_CHECKPOINT_EVENT                  = 161;
     /*
-    Gtid event. For global transaction ID, used to start a new event group,
-    instead of the old BEGIN query event, and also to mark stand-alone
-    events.
+     * Gtid event. For global transaction ID, used to start a new event group,
+     * instead of the old BEGIN query event, and also to mark stand-alone
+     * events.
      */
     public static final int    GTID_EVENT                               = 162;
     /*
-    Gtid list event. Logged at the start of every binlog, to record the
-    current replication state. This consists of the last GTID seen for
-    each replication domain.
+     * Gtid list event. Logged at the start of every binlog, to record the
+     * current replication state. This consists of the last GTID seen for each
+     * replication domain.
      */
     public static final int    GTID_LIST_EVENT                          = 163;
 
@@ -213,9 +193,8 @@ public abstract class LogEvent
     public static final int    ENUM_END_EVENT                           = 164;
 
     /**
-    1 byte length, 1 byte format
-    Length is total length in bytes, including 2 byte header
-    Length values 0 and 1 are currently invalid and reserved.
+     * 1 byte length, 1 byte format Length is total length in bytes, including 2
+     * byte header Length values 0 and 1 are currently invalid and reserved.
      */
     public static final int    EXTRA_ROW_INFO_LEN_OFFSET                = 0;
     public static final int    EXTRA_ROW_INFO_FORMAT_OFFSET             = 1;
@@ -260,11 +239,10 @@ public abstract class LogEvent
     public static final int    MARIA_SLAVE_CAPABILITY_MINE              = MARIA_SLAVE_CAPABILITY_GTID;
 
     /**
-        For an event, 'e', carrying a type code, that a slave,
-        's', does not recognize, 's' will check 'e' for
-        LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
-        is ignored. Otherwise, 's' acknowledges that it has
-        found an unknown event in the relay log.
+     * For an event, 'e', carrying a type code, that a slave, 's', does not
+     * recognize, 's' will check 'e' for LOG_EVENT_IGNORABLE_F, and if the flag
+     * is set, then 'e' is ignored. Otherwise, 's' acknowledges that it has
+     * found an unknown event in the relay log.
      */
     public static final int    LOG_EVENT_IGNORABLE_F                    = 0x80;
 
@@ -289,6 +267,7 @@ public abstract class LogEvent
     public static final int    MYSQL_TYPE_TIMESTAMP2                    = 17;
     public static final int    MYSQL_TYPE_DATETIME2                     = 18;
     public static final int    MYSQL_TYPE_TIME2                         = 19;
+    public static final int    MYSQL_TYPE_JSON                          = 245;
     public static final int    MYSQL_TYPE_NEWDECIMAL                    = 246;
     public static final int    MYSQL_TYPE_ENUM                          = 247;
     public static final int    MYSQL_TYPE_SET                           = 248;
@@ -300,10 +279,8 @@ public abstract class LogEvent
     public static final int    MYSQL_TYPE_STRING                        = 254;
     public static final int    MYSQL_TYPE_GEOMETRY                      = 255;
 
-    public static String getTypeName(final int type)
-    {
-        switch (type)
-        {
+    public static String getTypeName(final int type) {
+        switch (type) {
             case START_EVENT_V3:
                 return "Start_v3";
             case STOP_EVENT:
@@ -383,16 +360,14 @@ public abstract class LogEvent
 
     protected final LogHeader  header;
 
-    protected LogEvent(LogHeader header)
-    {
+    protected LogEvent(LogHeader header){
         this.header = header;
     }
 
     /**
      * Return event header.
      */
-    public final LogHeader getHeader()
-    {
+    public final LogHeader getHeader() {
         return header;
     }
 
@@ -400,16 +375,14 @@ public abstract class LogEvent
      * The total size of this event, in bytes. In other words, this is the sum
      * of the sizes of Common-Header, Post-Header, and Body.
      */
-    public final int getEventLen()
-    {
+    public final int getEventLen() {
         return header.getEventLen();
     }
 
     /**
      * Server ID of the server that created the event.
      */
-    public final long getServerId()
-    {
+    public final long getServerId() {
         return header.getServerId();
     }
 
@@ -420,18 +393,15 @@ public abstract class LogEvent
      * file. In a relay log, this is the position of the next event in the
      * master's binlog.
      */
-    public final long getLogPos()
-    {
+    public final long getLogPos() {
         return header.getLogPos();
     }
 
     /**
      * The time when the query started, in seconds since 1970.
      */
-    public final long getWhen()
-    {
+    public final long getWhen() {
         return header.getWhen();
     }
 
-    
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java

@@ -29,8 +29,8 @@ import java.util.Arrays;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class LogFetcher extends LogBuffer implements Closeable
-{
+public abstract class LogFetcher extends LogBuffer implements Closeable {
+
     /** Default initial capacity. */
     public static final int   DEFAULT_INITIAL_CAPACITY = 8192;
 
@@ -42,18 +42,15 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
 
     protected final float     factor;
 
-    public LogFetcher()
-    {
+    public LogFetcher(){
         this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
     }
 
-    public LogFetcher(final int initialCapacity)
-    {
+    public LogFetcher(final int initialCapacity){
         this(initialCapacity, DEFAULT_GROWTH_FACTOR);
     }
 
-    public LogFetcher(final int initialCapacity, final float growthFactor)
-    {
+    public LogFetcher(final int initialCapacity, final float growthFactor){
         this.buffer = new byte[initialCapacity];
         this.factor = growthFactor;
     }
@@ -65,15 +62,12 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
      * 
      * @param minCapacity the desired minimum capacity
      */
-    protected final void ensureCapacity(final int minCapacity)
-    {
+    protected final void ensureCapacity(final int minCapacity) {
         final int oldCapacity = buffer.length;
 
-        if (minCapacity > oldCapacity)
-        {
+        if (minCapacity > oldCapacity) {
             int newCapacity = (int) (oldCapacity * factor);
-            if (newCapacity < minCapacity)
-                newCapacity = minCapacity;
+            if (newCapacity < minCapacity) newCapacity = minCapacity;
 
             buffer = Arrays.copyOf(buffer, newCapacity);
         }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java

@@ -6,8 +6,8 @@ package com.taobao.tddl.dbsync.binlog;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class LogPosition implements Cloneable, Comparable<LogPosition>
-{
+public class LogPosition implements Cloneable, Comparable<LogPosition> {
+
     /* binlog file's name */
     protected String fileName;
 
@@ -19,8 +19,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @param fileName file name for binlog files: mysql-bin.000001
      */
-    public LogPosition(String fileName)
-    {
+    public LogPosition(String fileName){
         this.fileName = fileName;
         this.position = 0L;
     }
@@ -30,8 +29,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @param fileName file name for binlog files: mysql-bin.000001
      */
-    public LogPosition(String fileName, final long position)
-    {
+    public LogPosition(String fileName, final long position){
         this.fileName = fileName;
         this.position = position;
     }
@@ -39,31 +37,24 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
     /**
      * Binlog position copy init.
      */
-    public LogPosition(LogPosition source)
-    {
+    public LogPosition(LogPosition source){
         this.fileName = source.fileName;
         this.position = source.position;
     }
 
-    public final String getFileName()
-    {
+    public final String getFileName() {
         return fileName;
     }
 
-    public final long getPosition()
-    {
+    public final long getPosition() {
         return position;
     }
 
     /* Clone binlog position without CloneNotSupportedException */
-    public LogPosition clone()
-    {
-        try
-        {
+    public LogPosition clone() {
+        try {
             return (LogPosition) super.clone();
-        }
-        catch (CloneNotSupportedException e)
-        {
+        } catch (CloneNotSupportedException e) {
             // Never happend
             return null;
         }
@@ -72,12 +63,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
     /**
      * Compares with the specified fileName and position.
      */
-    public final int compareTo(String fileName, final long position)
-    {
+    public final int compareTo(String fileName, final long position) {
         final int val = this.fileName.compareTo(fileName);
 
-        if (val == 0)
-        {
+        if (val == 0) {
             return (int) (this.position - position);
         }
         return val;
@@ -88,12 +77,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Comparable#compareTo(java.lang.Object)
      */
-    public int compareTo(LogPosition o)
-    {
+    public int compareTo(LogPosition o) {
         final int val = fileName.compareTo(o.fileName);
 
-        if (val == 0)
-        {
+        if (val == 0) {
             return (int) (position - o.position);
         }
         return val;
@@ -104,13 +91,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Object#equals(java.lang.Object)
      */
-    public boolean equals(Object obj)
-    {
-        if (obj instanceof LogPosition)
-        {
+    public boolean equals(Object obj) {
+        if (obj instanceof LogPosition) {
             LogPosition pos = ((LogPosition) obj);
-            return fileName.equals(pos.fileName)
-                    && (this.position == pos.position);
+            return fileName.equals(pos.fileName) && (this.position == pos.position);
         }
         return false;
     }
@@ -120,8 +104,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Object#toString()
      */
-    public String toString()
-    {
+    public String toString() {
         return fileName + ':' + position;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java

@@ -9,8 +9,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class AppendBlockLogEvent extends LogEvent
-{
+public class AppendBlockLogEvent extends LogEvent {
+
     private final LogBuffer blockBuf;
     private final int       blockLen;
 
@@ -19,9 +19,7 @@ public class AppendBlockLogEvent extends LogEvent
     /* AB = "Append Block" */
     public static final int AB_FILE_ID_OFFSET = 0;
 
-    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -36,18 +34,15 @@ public class AppendBlockLogEvent extends LogEvent
         blockBuf = buffer.duplicate(blockLen);
     }
 
-    public final long getFileId()
-    {
+    public final long getFileId() {
         return fileId;
     }
 
-    public final LogBuffer getBuffer()
-    {
+    public final LogBuffer getBuffer() {
         return blockBuf;
     }
 
-    public final byte[] getData()
-    {
+    public final byte[] getData() {
         return blockBuf.getData();
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java

@@ -10,11 +10,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
-{
-    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent {
+
+    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java

@@ -8,8 +8,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class CreateFileLogEvent extends LoadLogEvent
-{
+public final class CreateFileLogEvent extends LoadLogEvent {
+
     protected LogBuffer     blockBuf;
     protected int           blockLen;
     protected long          fileId;
@@ -20,9 +20,7 @@ public final class CreateFileLogEvent extends LoadLogEvent
     public static final int CF_FILE_ID_OFFSET = 0;
     public static final int CF_DATA_OFFSET    = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
 
-    public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public CreateFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header, buffer, descriptionEvent);
 
         final int headerLen = descriptionEvent.commonHeaderLen;
@@ -30,44 +28,37 @@ public final class CreateFileLogEvent extends LoadLogEvent
         final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
 
         copyLogEvent(buffer,
-                ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
-                        : (headerLen + loadHeaderLen + createFileHeaderLen)),
+            ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen) : (headerLen + loadHeaderLen + createFileHeaderLen)),
             descriptionEvent);
 
-        if (descriptionEvent.binlogVersion != 1)
-        {
-            fileId = buffer.getUint32(headerLen + loadHeaderLen
-                    + CF_FILE_ID_OFFSET);
+        if (descriptionEvent.binlogVersion != 1) {
+            fileId = buffer.getUint32(headerLen + loadHeaderLen + CF_FILE_ID_OFFSET);
             /*
-              Note that it's ok to use get_data_size() below, because it is computed
-              with values we have already read from this event (because we called
-              copy_log_event()); we are not using slave's format info to decode
-              master's format, we are really using master's format info.
-              Anyway, both formats should be identical (except the common_header_len)
-              as these Load events are not changed between 4.0 and 5.0 (as logging of
-              LOAD DATA INFILE does not use Load_log_event in 5.0).
+             * Note that it's ok to use get_data_size() below, because it is
+             * computed with values we have already read from this event
+             * (because we called copy_log_event()); we are not using slave's
+             * format info to decode master's format, we are really using
+             * master's format info. Anyway, both formats should be identical
+             * (except the common_header_len) as these Load events are not
+             * changed between 4.0 and 5.0 (as logging of LOAD DATA INFILE does
+             * not use Load_log_event in 5.0).
              */
             blockLen = buffer.limit() - buffer.position();
             blockBuf = buffer.duplicate(blockLen);
-        }
-        else
-        {
+        } else {
             initedFromOld = true;
         }
     }
 
-    public final long getFileId()
-    {
+    public final long getFileId() {
         return fileId;
     }
 
-    public final LogBuffer getBuffer()
-    {
+    public final LogBuffer getBuffer() {
         return blockBuf;
     }
 
-    public final byte[] getData()
-    {
+    public final byte[] getData() {
         return blockBuf.getData();
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java

@@ -9,16 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteFileLogEvent extends LogEvent
-{
+public final class DeleteFileLogEvent extends LogEvent {
+
     private final long      fileId;
 
     /* DF = "Delete File" */
     public static final int DF_FILE_ID_OFFSET = 0;
 
-    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -26,8 +24,7 @@ public final class DeleteFileLogEvent extends LogEvent
         fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
     }
 
-    public final long getFileId()
-    {
+    public final long getFileId() {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteRowsLogEvent extends RowsLogEvent
-{
-    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+public final class DeleteRowsLogEvent extends RowsLogEvent {
+
+    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java

@@ -9,16 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class ExecuteLoadLogEvent extends LogEvent
-{
+public final class ExecuteLoadLogEvent extends LogEvent {
+
     private final long      fileId;
 
     /* EL = "Execute Load" */
     public static final int EL_FILE_ID_OFFSET = 0;
 
-    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -26,8 +24,7 @@ public final class ExecuteLoadLogEvent extends LogEvent
         fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
     }
 
-    public final long getFileId()
-    {
+    public final long getFileId() {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java

@@ -8,7 +8,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * Event responsible for LOAD DATA execution, it similar to Query_log_event but
  * before executing the query it substitutes original filename in LOAD DATA
  * query with name of temporary file.
- * 
  * <ul>
  * <li>4 bytes. The ID of the file to load.</li>
  * <li>4 bytes. The start position within the statement for filename
@@ -22,8 +21,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
-{
+public final class ExecuteLoadQueryLogEvent extends QueryLogEvent {
+
     /** file_id of temporary file */
     private long            fileId;
 
@@ -53,9 +52,8 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
     public static final int ELQ_FN_POS_END_OFFSET   = ELQ_FILE_ID_OFFSET + 8;
     public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
 
-    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent) throws IOException
-    {
+    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
+                                                                                                                   throws IOException{
         super(header, buffer, descriptionEvent);
 
         buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
@@ -66,36 +64,27 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
         dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
 
         final int len = query.length();
-        if (fnPosStart > len || fnPosEnd > len
-                || dupHandling > LOAD_DUP_REPLACE)
-        {
-            throw new IOException(String.format(
-                    "Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
-                            + "fn_pos_end=%d, dup_handling=%d", fnPosStart,
-                    fnPosEnd, dupHandling));
+        if (fnPosStart > len || fnPosEnd > len || dupHandling > LOAD_DUP_REPLACE) {
+            throw new IOException(String.format("Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
+                                                + "fn_pos_end=%d, dup_handling=%d", fnPosStart, fnPosEnd, dupHandling));
         }
     }
 
-    public final int getFilenamePosStart()
-    {
+    public final int getFilenamePosStart() {
         return fnPosStart;
     }
 
-    public final int getFilenamePosEnd()
-    {
+    public final int getFilenamePosEnd() {
         return fnPosEnd;
     }
 
-    public final String getFilename()
-    {
-        if (query != null)
-            return query.substring(fnPosStart, fnPosEnd).trim();
+    public final String getFilename() {
+        if (query != null) return query.substring(fnPosStart, fnPosEnd).trim();
 
         return null;
     }
 
-    public final long getFileId()
-    {
+    public final long getFileId() {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java

@@ -5,8 +5,8 @@ import java.io.IOException;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
 /**
- * For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
- * ordinary events).
+ * For binlog version 4. This event is saved by threads which read it, as they
+ * need it for future use (to decode the ordinary events).
  * 
  * @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
@@ -15,8 +15,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
     /**
-     * The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
-     * exist in binlogs, it does not have a format).
+     * The number of types we handle in Format_description_log_event
+     * (UNKNOWN_EVENT is not to be handled, it does not exist in binlogs, it
+     * does not have a format).
      */
     public static final int   LOG_EVENT_TYPES                     = (ENUM_END_EVENT - 1);
 
@@ -70,8 +71,9 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
     public static final long  checksumVersionProduct              = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
                                                                     * 256 + checksumVersionSplit[2];
     /**
-     * The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
-     * LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
+     * The size of the fixed header which _all_ events have (for binlogs written
+     * by this version, this is equal to LOG_EVENT_HEADER_LEN), except
+     * FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
      * LOG_EVENT_MINIMAL_HEADER_LEN).
      */
     protected final int       commonHeaderLen;
@@ -105,7 +107,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         calcServerVersionSplit();
         long calc = getVersionProduct();
         if (calc >= checksumVersionProduct) {
-            /* the last bytes are the checksum alg desc and value (or value's room) */
+            /*
+             * the last bytes are the checksum alg desc and value (or value's
+             * room)
+             */
             numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
         }
 
@@ -136,6 +141,11 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         }
     }
 
+    public FormatDescriptionLogEvent(final int binlogVersion, int binlogChecksum){
+        this(binlogVersion);
+        this.header.checksumAlg = binlogChecksum;
+    }
+
     public FormatDescriptionLogEvent(final int binlogVersion){
         this.binlogVersion = binlogVersion;
 
@@ -147,7 +157,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 commonHeaderLen = LOG_EVENT_HEADER_LEN;
                 numberOfEventTypes = LOG_EVENT_TYPES;
 
-                /* Note: all event types must explicitly fill in their lengths here. */
+                /*
+                 * Note: all event types must explicitly fill in their lengths
+                 * here.
+                 */
                 postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
                 postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
                 postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
@@ -171,11 +184,14 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 /*
-                 * We here have the possibility to simulate a master of before we changed the table map id to be stored
-                 * in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
-                 * compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
-                 * that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
-                 * test (rpl_row_4_bytes) too.
+                 * We here have the possibility to simulate a master of before
+                 * we changed the table map id to be stored in 6 bytes: when it
+                 * was stored in 4 bytes (=> post_header_len was 6). This is
+                 * used to test backward compatibility. This code can be removed
+                 * after a few months (today is Dec 21st 2005), when we know
+                 * that the 4-byte masters are not deployed anymore (check with
+                 * Tomas Ulin first!), and the accompanying test
+                 * (rpl_row_4_bytes) too.
                  */
                 postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
                 postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
@@ -195,17 +211,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
             case 3: /* 4.0.x x>=2 */
                 /*
-                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
-                 * versions send.
+                 * We build an artificial (i.e. not sent by the master) event,
+                 * which describes what those old master versions send.
                  */
                 serverVersion = "4.0";
                 commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
 
                 /*
-                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
-                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
-                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
-                 * detect less corruptions).
+                 * The first new event in binlog version 4 is Format_desc. So
+                 * any event type after that does not exist in older versions.
+                 * We use the events known by version 3, even if version 1 had
+                 * only a subset of them (this is not a problem: it uses a few
+                 * bytes for nothing but unifies code; it does not make the
+                 * slave detect less corruptions).
                  */
                 numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
 
@@ -222,17 +240,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
             case 1: /* 3.23 */
                 /*
-                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
-                 * versions send.
+                 * We build an artificial (i.e. not sent by the master) event,
+                 * which describes what those old master versions send.
                  */
                 serverVersion = "3.23";
                 commonHeaderLen = OLD_HEADER_LEN;
 
                 /*
-                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
-                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
-                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
-                 * detect less corruptions).
+                 * The first new event in binlog version 4 is Format_desc. So
+                 * any event type after that does not exist in older versions.
+                 * We use the events known by version 3, even if version 1 had
+                 * only a subset of them (this is not a problem: it uses a few
+                 * bytes for nothing but unifies code; it does not make the
+                 * slave detect less corruptions).
                  */
                 numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java

@@ -6,19 +6,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * <pre>
  *   Base class for ignorable log events. Events deriving from
-  this class can be safely ignored by slaves that cannot
-  recognize them. Newer slaves, will be able to read and
-  handle them. This has been designed to be an open-ended
-  architecture, so adding new derived events shall not harm
-  the old slaves that support ignorable log event mechanism
-  (they will just ignore unrecognized ignorable events).
-
-  @note The only thing that makes an event ignorable is that it has
-  the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
-  that ignorable event types derive from Ignorable_log_event; they may
-  just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
-  argument to the Log_event constructor.
-  </pre>
+ *   this class can be safely ignored by slaves that cannot
+ *   recognize them. Newer slaves, will be able to read and
+ *   handle them. This has been designed to be an open-ended
+ *   architecture, so adding new derived events shall not harm
+ *   the old slaves that support ignorable log event mechanism
+ *   (they will just ignore unrecognized ignorable events).
+ * 
+ *   @note The only thing that makes an event ignorable is that it has
+ *   the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
+ *   that ignorable event types derive from Ignorable_log_event; they may
+ *   just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
+ *   argument to the Log_event constructor.
+ * </pre>
  * 
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java

@@ -5,12 +5,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * Class representing an incident, an occurance out of the ordinary, that
- * happened on the master.
- * 
- * The event is used to inform the slave that something out of the ordinary
- * happened on the master that might cause the database to be in an inconsistent
- * state.
- * 
+ * happened on the master. The event is used to inform the slave that something
+ * out of the ordinary happened on the master that might cause the database to
+ * be in an inconsistent state.
  * <table id="IncidentFormat">
  * <caption>Incident event format</caption>
  * <tr>
@@ -38,8 +35,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class IncidentLogEvent extends LogEvent
-{
+public final class IncidentLogEvent extends LogEvent {
+
     public static final int INCIDENT_NONE        = 0;
 
     /** There are possibly lost events in the replication stream */
@@ -51,9 +48,7 @@ public final class IncidentLogEvent extends LogEvent
     private final int       incident;
     private final String    message;
 
-    public IncidentLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public IncidentLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -61,8 +56,7 @@ public final class IncidentLogEvent extends LogEvent
 
         buffer.position(commonHeaderLen);
         final int incidentNumber = buffer.getUint16();
-        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
-        {
+        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE) {
             // If the incident is not recognized, this binlog event is
             // invalid. If we set incident_number to INCIDENT_NONE, the
             // invalidity will be detected by is_valid().
@@ -76,13 +70,11 @@ public final class IncidentLogEvent extends LogEvent
         message = buffer.getString();
     }
 
-    public final int getIncident()
-    {
+    public final int getIncident() {
         return incident;
     }
 
-    public final String getMessage()
-    {
+    public final String getMessage() {
         return message;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java

@@ -6,54 +6,43 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * An Intvar_log_event will be created just before a Query_log_event, if the
  * query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
- * Intvar_log_event holds the value of one of these variables.
- * 
- * Binary Format
- * 
- * The Post-Header for this event type is empty. The Body has two components:
- * 
+ * Intvar_log_event holds the value of one of these variables. Binary Format The
+ * Post-Header for this event type is empty. The Body has two components:
  * <table>
  * <caption>Body for Intvar_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>type</td>
  * <td>1 byte enumeration</td>
  * <td>One byte identifying the type of variable stored. Currently, two
  * identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
  * </tr>
- * 
  * <tr>
  * <td>value</td>
  * <td>8 byte unsigned integer</td>
  * <td>The value of the variable.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class IntvarLogEvent extends LogEvent
-{
+public final class IntvarLogEvent extends LogEvent {
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
      * 1 or INSERT_ID_EVENT = 2.</li>
      * <li>8 bytes. An unsigned integer indicating the value to be used for the
      * LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long      value;
@@ -68,31 +57,25 @@ public final class IntvarLogEvent extends LogEvent
     public static final int LAST_INSERT_ID_EVENT = 1;
     public static final int INSERT_ID_EVENT      = 2;
 
-    public IntvarLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public IntvarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         /* The Post-Header is empty. The Varible Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
-                + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
                         + I_TYPE_OFFSET);
         type = buffer.getInt8(); // I_TYPE_OFFSET
         value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
     }
 
-    public final int getType()
-    {
+    public final int getType() {
         return type;
     }
 
-    public final long getValue()
-    {
+    public final long getValue() {
         return value;
     }
 
-    public final String getQuery()
-    {
+    public final String getQuery() {
         return "SET INSERT_ID = " + value;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java

@@ -26,16 +26,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * </pre>
  * 
  * Binary Format: The Post-Header consists of the following six components.
- * 
  * <table>
  * <caption>Post-Header for Load_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>slave_proxy_id</td>
  * <td>4 byte unsigned integer</td>
@@ -45,71 +42,57 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * temporary table local to the client. The slave_proxy_id is used to
  * distinguish temporary tables that belong to different clients.</td>
  * </tr>
- * 
  * <tr>
  * <td>exec_time</td>
  * <td>4 byte unsigned integer</td>
  * <td>The time from when the query started to when it was logged in the binlog,
  * in seconds.</td>
  * </tr>
- * 
  * <tr>
  * <td>skip_lines</td>
  * <td>4 byte unsigned integer</td>
  * <td>The number on line (14) above, if present, or 0 if line (14) is left out.
  * </td>
  * </tr>
- * 
  * <tr>
  * <td>table_name_len</td>
  * <td>1 byte unsigned integer</td>
  * <td>The length of 'table_name' on line (4) above.</td>
  * </tr>
- * 
  * <tr>
  * <td>db_len</td>
  * <td>1 byte unsigned integer</td>
  * <td>The length of 'db' on line (1) above.</td>
  * </tr>
- * 
  * <tr>
  * <td>num_fields</td>
  * <td>4 byte unsigned integer</td>
  * <td>The number n of fields on line (15) above.</td>
  * </tr>
  * </table>
- * 
  * The Body contains the following components.
- * 
  * <table>
  * <caption>Body of Load_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>sql_ex</td>
  * <td>variable length</td>
- * 
  * <td>Describes the part of the query on lines (3) and (5)&ndash;(13) above.
  * More precisely, it stores the five strings (on lines) field_term (6),
  * enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
  * bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
- * OPTIONALLY (7).
- * 
- * The data is stored in one of two formats, called "old" and "new". The type
- * field of Common-Header determines which of these two formats is used: type
- * LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
- * that the new format is used. When MySQL writes a Load_log_event, it uses the
- * new format if at least one of the five strings is two or more bytes long.
- * Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
- * used.
- * 
- * The new and old format differ in the way the five strings are stored.
- * 
+ * OPTIONALLY (7). The data is stored in one of two formats, called "old" and
+ * "new". The type field of Common-Header determines which of these two formats
+ * is used: type LOAD_EVENT means that the old format is used, and type
+ * NEW_LOAD_EVENT means that the new format is used. When MySQL writes a
+ * Load_log_event, it uses the new format if at least one of the five strings is
+ * two or more bytes long. Otherwise (i.e., if all strings are 0 or 1 bytes
+ * long), the old format is used. The new and old format differ in the way the
+ * five strings are stored.
  * <ul>
  * <li>In the new format, the strings are stored in the order field_term,
  * enclosed, escaped, line_term, line_start. Each string consists of a length (1
@@ -117,7 +100,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
  * IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
  * the presence of the corresponding keyword in the SQL query.
- * 
  * <li>In the old format, we know that each string has length 0 or 1. Therefore,
  * only the first byte of each string is stored. The order of the strings is the
  * same as in the new format. These five bytes are followed by the same 1 byte
@@ -128,18 +110,15 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
  * LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
  * </ul>
- * 
  * Thus, the size of the new format is 6 bytes + the sum of the sizes of the
  * five strings. The size of the old format is always 7 bytes.</td>
  * </tr>
- * 
  * <tr>
  * <td>field_lens</td>
  * <td>num_fields 1 byte unsigned integers</td>
  * <td>An array of num_fields integers representing the length of each field in
  * the query. (num_fields is from the Post-Header).</td>
  * </tr>
- * 
  * <tr>
  * <td>fields</td>
  * <td>num_fields null-terminated strings</td>
@@ -148,7 +127,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * the num_fields array.) The total length of all strings equals to the sum of
  * all field_lens, plus num_fields bytes for all the trailing zeros.</td>
  * </tr>
- * 
  * <tr>
  * <td>table_name</td>
  * <td>null-terminated string of length table_len+1 bytes</td>
@@ -156,31 +134,27 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * trailing zero is actually redundant since the table_len is known from
  * Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>db</td>
  * <td>null-terminated string of length db_len+1 bytes</td>
  * <td>The 'db' from the query, as a null-terminated string. (The trailing zero
  * is actually redundant since the db_len is known from Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>file_name</td>
  * <td>variable length string without trailing zero, extending to the end of the
  * event (determined by the length field of the Common-Header)</td>
  * <td>The 'file_name' from the query.</td>
  * </tr>
- * 
  * </table>
- * 
  * This event type is understood by current versions, but only generated by
  * MySQL 3.23 and earlier.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class LoadLogEvent extends LogEvent
-{
+public class LoadLogEvent extends LogEvent {
+
     private String          table;
     private String          db;
     private String          fname;
@@ -210,10 +184,9 @@ public class LoadLogEvent extends LogEvent
     public static final int L_DATA_OFFSET       = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
 
     /*
-      These are flags and structs to handle all the LOAD DATA INFILE options (LINES
-      TERMINATED etc).
-      DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
-      DATA).
+     * These are flags and structs to handle all the LOAD DATA INFILE options
+     * (LINES TERMINATED etc). DUMPFILE_FLAG is probably useless (DUMPFILE is a
+     * clause of SELECT, not of LOAD DATA).
      */
     public static final int DUMPFILE_FLAG       = 0x1;
     public static final int OPT_ENCLOSED_FLAG   = 0x2;
@@ -226,9 +199,7 @@ public class LoadLogEvent extends LogEvent
     public static final int LINE_START_EMPTY    = 0x8;
     public static final int ESCAPED_EMPTY       = 0x10;
 
-    public LoadLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public LoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
@@ -236,8 +207,8 @@ public class LoadLogEvent extends LogEvent
          * I (Guilhem) manually tested replication of LOAD DATA INFILE for
          * 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
          */
-        copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
-                + descriptionEvent.commonHeaderLen : loadHeaderLen
+        copyLogEvent(buffer,
+            ((header.type == LOAD_EVENT) ? loadHeaderLen + descriptionEvent.commonHeaderLen : loadHeaderLen
                                                                                               + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
             descriptionEvent);
     }
@@ -245,9 +216,7 @@ public class LoadLogEvent extends LogEvent
     /**
      * @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
      */
-    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset, FormatDescriptionLogEvent descriptionEvent) {
         /* this is the beginning of the post-header */
         buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
 
@@ -262,8 +231,7 @@ public class LoadLogEvent extends LogEvent
          * Sql_ex.init() on success returns the pointer to the first byte after
          * the sql_ex structure, which is the start of field lengths array.
          */
-        if (header.type != LOAD_EVENT /* use_new_format */)
-        {
+        if (header.type != LOAD_EVENT /* use_new_format */) {
             /*
              * The code below assumes that buf will not disappear from under our
              * feet during the lifetime of the event. This assumption holds true
@@ -279,9 +247,7 @@ public class LoadLogEvent extends LogEvent
             escaped = buffer.getString();
             optFlags = buffer.getInt8();
             emptyFlags = 0;
-        }
-        else
-        {
+        } else {
             fieldTerm = buffer.getFixString(1);
             enclosed = buffer.getFixString(1);
             lineTerm = buffer.getFixString(1);
@@ -290,23 +256,17 @@ public class LoadLogEvent extends LogEvent
             optFlags = buffer.getUint8();
             emptyFlags = buffer.getUint8();
 
-            if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
-                fieldTerm = null;
-            if ((emptyFlags & ENCLOSED_EMPTY) != 0)
-                enclosed = null;
-            if ((emptyFlags & LINE_TERM_EMPTY) != 0)
-                lineTerm = null;
-            if ((emptyFlags & LINE_START_EMPTY) != 0)
-                lineStart = null;
-            if ((emptyFlags & ESCAPED_EMPTY) != 0)
-                escaped = null;
+            if ((emptyFlags & FIELD_TERM_EMPTY) != 0) fieldTerm = null;
+            if ((emptyFlags & ENCLOSED_EMPTY) != 0) enclosed = null;
+            if ((emptyFlags & LINE_TERM_EMPTY) != 0) lineTerm = null;
+            if ((emptyFlags & LINE_START_EMPTY) != 0) lineStart = null;
+            if ((emptyFlags & ESCAPED_EMPTY) != 0) escaped = null;
         }
 
         final int fieldLenPos = buffer.position();
         buffer.forward(numFields);
         fields = new String[numFields];
-        for (int i = 0; i < numFields; i++)
-        {
+        for (int i = 0; i < numFields; i++) {
             final int fieldLen = buffer.getUint8(fieldLenPos + i);
             fields[i] = buffer.getFixString(fieldLen + 1);
         }
@@ -324,68 +284,55 @@ public class LoadLogEvent extends LogEvent
         buffer.forward(1); // The + 1 is for \0 terminating fname
     }
 
-    public final String getTable()
-    {
+    public final String getTable() {
         return table;
     }
 
-    public final String getDb()
-    {
+    public final String getDb() {
         return db;
     }
 
-    public final String getFname()
-    {
+    public final String getFname() {
         return fname;
     }
 
-    public final int getSkipLines()
-    {
+    public final int getSkipLines() {
         return skipLines;
     }
 
-    public final String[] getFields()
-    {
+    public final String[] getFields() {
         return fields;
     }
 
-    public final String getFieldTerm()
-    {
+    public final String getFieldTerm() {
         return fieldTerm;
     }
 
-    public final String getLineTerm()
-    {
+    public final String getLineTerm() {
         return lineTerm;
     }
 
-    public final String getLineStart()
-    {
+    public final String getLineStart() {
         return lineStart;
     }
 
-    public final String getEnclosed()
-    {
+    public final String getEnclosed() {
         return enclosed;
     }
 
-    public final String getEscaped()
-    {
+    public final String getEscaped() {
         return escaped;
     }
 
-    public final int getOptFlags()
-    {
+    public final int getOptFlags() {
         return optFlags;
     }
 
-    public final int getEmptyFlags()
-    {
+    public final int getEmptyFlags() {
         return emptyFlags;
     }
 
-    public final long getExecTime()
-    {
+    public final long getExecTime() {
         return execTime;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java

@@ -11,41 +11,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * different format and length even for different events of the same type. The
  * binary formats of Post-Header and Body are documented separately in each
  * subclass. The binary format of Common-Header is as follows.
- * 
  * <table>
  * <caption>Common-Header</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>timestamp</td>
  * <td>4 byte unsigned integer</td>
  * <td>The time when the query started, in seconds since 1970.</td>
  * </tr>
- * 
  * <tr>
  * <td>type</td>
  * <td>1 byte enumeration</td>
  * <td>See enum #Log_event_type.</td>
  * </tr>
- * 
  * <tr>
  * <td>server_id</td>
  * <td>4 byte unsigned integer</td>
  * <td>Server ID of the server that created the event.</td>
  * </tr>
- * 
  * <tr>
  * <td>total_size</td>
  * <td>4 byte unsigned integer</td>
  * <td>The total size of this event, in bytes. In other words, this is the sum
  * of the sizes of Common-Header, Post-Header, and Body.</td>
  * </tr>
- * 
  * <tr>
  * <td>master_position</td>
  * <td>4 byte unsigned integer</td>
@@ -54,24 +47,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * the position of the next event, in bytes from the beginning of the file. In a
  * relay log, this is the position of the next event in the master's binlog.</td>
  * </tr>
- * 
  * <tr>
  * <td>flags</td>
  * <td>2 byte bitfield</td>
  * <td>See Log_event::flags.</td>
  * </tr>
  * </table>
- * 
  * Summing up the numbers above, we see that the total size of the common header
  * is 19 bytes.
  * 
  * @see mysql-5.1.60/sql/log_event.cc
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogHeader
-{
+public final class LogHeader {
+
     protected final int type;
 
     /**
@@ -114,34 +104,29 @@ public final class LogHeader
 
     /**
      * The value is set by caller of FD constructor and
-     * Log_event::write_header() for the rest.
-     * In the FD case it's propagated into the last byte 
-     * of post_header_len[] at FD::write().
-     * On the slave side the value is assigned from post_header_len[last] 
-     * of the last seen FD event.
+     * Log_event::write_header() for the rest. In the FD case it's propagated
+     * into the last byte of post_header_len[] at FD::write(). On the slave side
+     * the value is assigned from post_header_len[last] of the last seen FD
+     * event.
      */
     protected int       checksumAlg;
     /**
-       Placeholder for event checksum while writing to binlog.
+     * Placeholder for event checksum while writing to binlog.
      */
     protected long      crc;        // ha_checksum
 
     /* for Start_event_v3 */
-    public LogHeader(final int type)
-    {
+    public LogHeader(final int type){
         this.type = type;
     }
 
-    public LogHeader(LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public LogHeader(LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         when = buffer.getUint32();
         type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET;
         serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET;
         eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET;
 
-        if (descriptionEvent.binlogVersion == 1)
-        {
+        if (descriptionEvent.binlogVersion == 1) {
             logPos = 0;
             flags = 0;
             return;
@@ -150,48 +135,43 @@ public final class LogHeader
         /* 4.0 or newer */
         logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET
         /*
-          If the log is 4.0 (so here it can only be a 4.0 relay log read by
-          the SQL thread or a 4.0 master binlog read by the I/O thread),
-          log_pos is the beginning of the event: we transform it into the end
-          of the event, which is more useful.
-          But how do you know that the log is 4.0: you know it if
-          description_event is version 3 *and* you are not reading a
-          Format_desc (remember that mysqlbinlog starts by assuming that 5.0
-          logs are in 4.0 format, until it finds a Format_desc).
+         * If the log is 4.0 (so here it can only be a 4.0 relay log read by the
+         * SQL thread or a 4.0 master binlog read by the I/O thread), log_pos is
+         * the beginning of the event: we transform it into the end of the
+         * event, which is more useful. But how do you know that the log is 4.0:
+         * you know it if description_event is version 3 *and* you are not
+         * reading a Format_desc (remember that mysqlbinlog starts by assuming
+         * that 5.0 logs are in 4.0 format, until it finds a Format_desc).
          */
-        if (descriptionEvent.binlogVersion == 3
-                && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0)
-        {
+        if (descriptionEvent.binlogVersion == 3 && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0) {
             /*
-              If log_pos=0, don't change it. log_pos==0 is a marker to mean
-              "don't change rli->group_master_log_pos" (see
-              inc_group_relay_log_pos()). As it is unreal log_pos, adding the
-              event len's is nonsense. For example, a fake Rotate event should
-              not have its log_pos (which is 0) changed or it will modify
-              Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
-              value of (a non-zero offset which does not exist in the master's
-              binlog, so which will cause problems if the user uses this value
-              in CHANGE MASTER).
+             * If log_pos=0, don't change it. log_pos==0 is a marker to mean
+             * "don't change rli->group_master_log_pos" (see
+             * inc_group_relay_log_pos()). As it is unreal log_pos, adding the
+             * event len's is nonsense. For example, a fake Rotate event should
+             * not have its log_pos (which is 0) changed or it will modify
+             * Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
+             * value of (a non-zero offset which does not exist in the master's
+             * binlog, so which will cause problems if the user uses this value
+             * in CHANGE MASTER).
              */
             logPos += eventLen; /* purecov: inspected */
         }
 
         flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET
-        if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT)
-                || (type == LogEvent.ROTATE_EVENT))
-        {
+        if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT) || (type == LogEvent.ROTATE_EVENT)) {
             /*
-              These events always have a header which stops here (i.e. their
-              header is FROZEN).
+             * These events always have a header which stops here (i.e. their
+             * header is FROZEN).
              */
             /*
-              Initialization to zero of all other Log_event members as they're
-              not specified. Currently there are no such members; in the future
-              there will be an event UID (but Format_description and Rotate
-              don't need this UID, as they are not propagated through
-              --log-slave-updates (remember the UID is used to not play a query
-              twice when you have two masters which are slaves of a 3rd master).
-              Then we are done.
+             * Initialization to zero of all other Log_event members as they're
+             * not specified. Currently there are no such members; in the future
+             * there will be an event UID (but Format_description and Rotate
+             * don't need this UID, as they are not propagated through
+             * --log-slave-updates (remember the UID is used to not play a query
+             * twice when you have two masters which are slaves of a 3rd
+             * master). Then we are done.
              */
 
             if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) {
@@ -213,25 +193,22 @@ public final class LogHeader
         }
 
         /*
-        CRC verification by SQL and Show-Binlog-Events master side.
-        The caller has to provide @description_event->checksum_alg to
-        be the last seen FD's (A) descriptor.
-        If event is FD the descriptor is in it.
-        Notice, FD of the binlog can be only in one instance and therefore
-        Show-Binlog-Events executing master side thread needs just to know
-        the only FD's (A) value -  whereas RL can contain more.
-        In the RL case, the alg is kept in FD_e (@description_event) which is reset 
-        to the newer read-out event after its execution with possibly new alg descriptor.
-        Therefore in a typical sequence of RL:
-        {FD_s^0, FD_m, E_m^1} E_m^1 
-        will be verified with (A) of FD_m.
-
-        See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs
-        lines (log.h).
-
-        Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF.
+         * CRC verification by SQL and Show-Binlog-Events master side. The
+         * caller has to provide @description_event->checksum_alg to be the last
+         * seen FD's (A) descriptor. If event is FD the descriptor is in it.
+         * Notice, FD of the binlog can be only in one instance and therefore
+         * Show-Binlog-Events executing master side thread needs just to know
+         * the only FD's (A) value - whereas RL can contain more. In the RL
+         * case, the alg is kept in FD_e (@description_event) which is reset to
+         * the newer read-out event after its execution with possibly new alg
+         * descriptor. Therefore in a typical sequence of RL: {FD_s^0, FD_m,
+         * E_m^1} E_m^1 will be verified with (A) of FD_m. See legends
+         * definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs lines
+         * (log.h). Notice, a pre-checksum FD version forces alg :=
+         * BINLOG_CHECKSUM_ALG_UNDEF.
          */
-        checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg
+        checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch
+                                                                // checksum alg
         processCheckSum(buffer);
         /* otherwise, go on with reading the header from buf (nothing now) */
     }
@@ -239,8 +216,7 @@ public final class LogHeader
     /**
      * The different types of log events.
      */
-    public final int getType()
-    {
+    public final int getType() {
         return type;
     }
 
@@ -251,8 +227,7 @@ public final class LogHeader
      * file. In a relay log, this is the position of the next event in the
      * master's binlog.
      */
-    public final long getLogPos()
-    {
+    public final long getLogPos() {
         return logPos;
     }
 
@@ -260,24 +235,21 @@ public final class LogHeader
      * The total size of this event, in bytes. In other words, this is the sum
      * of the sizes of Common-Header, Post-Header, and Body.
      */
-    public final int getEventLen()
-    {
+    public final int getEventLen() {
         return eventLen;
     }
 
     /**
      * The time when the query started, in seconds since 1970.
      */
-    public final long getWhen()
-    {
+    public final long getWhen() {
         return when;
     }
 
     /**
      * Server ID of the server that created the event.
      */
-    public final long getServerId()
-    {
+    public final long getServerId() {
         return serverId;
     }
 
@@ -286,24 +258,20 @@ public final class LogHeader
      * LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
      * LOG_EVENT_SUPPRESS_USE_F for notes.
      */
-    public final int getFlags()
-    {
+    public final int getFlags() {
         return flags;
     }
 
-    
     public long getCrc() {
         return crc;
     }
 
-    
     public int getChecksumAlg() {
         return checksumAlg;
     }
 
     private void processCheckSum(LogBuffer buffer) {
-        if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF &&
-                checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){
+        if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
             crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
         }
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java

@@ -4,7 +4,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * 
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3
  * @since mysql 5.6
@@ -16,4 +15,3 @@ public class PreviousGtidsLogEvent  extends LogEvent{
         // do nothing , just for mysql gtid search function
     }
 }
-

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java

@@ -9,19 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * A Query_log_event is created for each query that modifies the database,
- * unless the query is logged row-based.
- * 
- * The Post-Header has five components:
- * 
+ * unless the query is logged row-based. The Post-Header has five components:
  * <table>
  * <caption>Post-Header for Query_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>slave_proxy_id</td>
  * <td>4 byte unsigned integer</td>
@@ -31,20 +26,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * temporary table local to the client. The slave_proxy_id is used to
  * distinguish temporary tables that belong to different clients.</td>
  * </tr>
- * 
  * <tr>
  * <td>exec_time</td>
  * <td>4 byte unsigned integer</td>
  * <td>The time from when the query started to when it was logged in the binlog,
  * in seconds.</td>
  * </tr>
- * 
  * <tr>
  * <td>db_len</td>
  * <td>1 byte integer</td>
  * <td>The length of the name of the currently selected database.</td>
  * </tr>
- * 
  * <tr>
  * <td>error_code</td>
  * <td>2 byte unsigned integer</td>
@@ -52,7 +44,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * fail with the same error code, except for the error codes ER_DB_CREATE_EXISTS
  * == 1007 and ER_DB_DROP_EXISTS == 1008.</td>
  * </tr>
- * 
  * <tr>
  * <td>status_vars_len</td>
  * <td>2 byte unsigned integer</td>
@@ -60,18 +51,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * query_log_event_status_vars "below".</td>
  * </tr>
  * </table>
- * 
  * The Body has the following components:
- * 
  * <table>
  * <caption>Body for Query_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>query_log_event_status_vars status_vars</td>
  * <td>status_vars_len bytes</td>
@@ -81,16 +68,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * Table_query_log_event_status_vars "below". MySQL always writes events in the
  * order defined below; however, it is capable of reading them in any order.</td>
  * </tr>
- * 
  * <tr>
  * <td>db</td>
  * <td>db_len+1</td>
- * <td>The currently selected database, as a null-terminated string.
- * 
- * (The trailing zero is redundant since the length is already known; it is
- * db_len from Post-Header.)</td>
+ * <td>The currently selected database, as a null-terminated string. (The
+ * trailing zero is redundant since the length is already known; it is db_len
+ * from Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>query</td>
  * <td>variable length string without trailing zero, extending to the end of the
@@ -98,20 +82,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>The SQL query.</td>
  * </tr>
  * </table>
- * 
  * The following table lists the status variables that may appear in the
  * status_vars field. Table_query_log_event_status_vars
- * 
  * <table>
  * <caption>Status variables for Query_log_event</caption>
- * 
  * <tr>
  * <th>Status variable</th>
  * <th>1 byte identifier</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>flags2</td>
  * <td>Q_FLAGS2_CODE == 0</td>
@@ -121,16 +101,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * identifies those options that need to be written to the binlog (not all do).
  * Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL |
  * OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
- * OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex.
- * 
- * These flags correspond to the SQL variables SQL_AUTO_IS_NULL,
- * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and AUTOCOMMIT, documented in the
- * "SET Syntax" section of the MySQL Manual.
- * 
- * This field is always written to the binlog in version >= 5.0, and never
- * written in version < 5.0.</td>
+ * OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. These flags correspond to the
+ * SQL variables SQL_AUTO_IS_NULL, FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and
+ * AUTOCOMMIT, documented in the "SET Syntax" section of the MySQL Manual. This
+ * field is always written to the binlog in version >= 5.0, and never written in
+ * version < 5.0.</td>
  * </tr>
- * 
  * <tr>
  * <td>sql_mode</td>
  * <td>Q_SQL_MODE_CODE == 1</td>
@@ -177,11 +153,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * All these flags are replicated from the server. However, all flags except
  * MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always preserves
  * its old value of MODE_NO_DIR_IN_CREATE. For a rationale, see comment in
- * Query_log_event::do_apply_event in log_event.cc.
- * 
- * This field is always written to the binlog.</td>
+ * Query_log_event::do_apply_event in log_event.cc. This field is always written
+ * to the binlog.</td>
  * </tr>
- * 
  * <tr>
  * <td>catalog</td>
  * <td>Q_CATALOG_NZ_CODE == 6</td>
@@ -189,23 +163,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * characters (at most 255 bytes)</td>
  * <td>Stores the client's current catalog. Every database belongs to a catalog,
  * the same way that every table belongs to a database. Currently, there is only
- * one catalog, "std".
- * 
- * This field is written if the length of the catalog is > 0; otherwise it is
- * not written.</td>
+ * one catalog, "std". This field is written if the length of the catalog is >
+ * 0; otherwise it is not written.</td>
  * </tr>
- * 
  * <tr>
  * <td>auto_increment</td>
  * <td>Q_AUTO_INCREMENT == 3</td>
  * <td>two 2 byte unsigned integers, totally 2+2=4 bytes</td>
- * 
  * <td>The two variables auto_increment_increment and auto_increment_offset, in
  * that order. For more information, see "System variables" in the MySQL manual.
- * 
  * This field is written if auto_increment > 1. Otherwise, it is not written.</td>
  * </tr>
- * 
  * <tr>
  * <td>charset</td>
  * <td>Q_CHARSET_CODE == 4</td>
@@ -216,33 +184,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * collation_connection identifies the character set and collation that the
  * master converts the query to when it receives it; this is useful when
  * comparing literal strings. collation_server is the default character set and
- * collation used when a new database is created.
- * 
- * See also "Connection Character Sets and Collations" in the MySQL 5.1 manual.
- * 
- * All three variables are codes identifying a (character set, collation) pair.
- * To see which codes map to which pairs, run the query "SELECT id,
- * character_set_name, collation_name FROM COLLATIONS".
- * 
- * Cf. Q_CHARSET_DATABASE_CODE below.
- * 
- * This field is always written.</td>
+ * collation used when a new database is created. See also
+ * "Connection Character Sets and Collations" in the MySQL 5.1 manual. All three
+ * variables are codes identifying a (character set, collation) pair. To see
+ * which codes map to which pairs, run the query "SELECT id, character_set_name,
+ * collation_name FROM COLLATIONS". Cf. Q_CHARSET_DATABASE_CODE below. This
+ * field is always written.</td>
  * </tr>
- * 
  * <tr>
  * <td>time_zone</td>
  * <td>Q_TIME_ZONE_CODE == 5</td>
  * <td>Variable-length string: the length in bytes (1 byte) followed by the
  * characters (at most 255 bytes).
- * <td>The time_zone of the master.
- * 
- * See also "System Variables" and "MySQL Server Time Zone Support" in the MySQL
- * manual.
- * 
- * This field is written if the length of the time zone string is > 0;
- * otherwise, it is not written.</td>
+ * <td>The time_zone of the master. See also "System Variables" and
+ * "MySQL Server Time Zone Support" in the MySQL manual. This field is written
+ * if the length of the time zone string is > 0; otherwise, it is not written.</td>
  * </tr>
- * 
  * <tr>
  * <td>lc_time_names_number</td>
  * <td>Q_LC_TIME_NAMES_CODE == 7</td>
@@ -251,84 +208,68 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * to languages is defined in sql_locale.cc. This field is written if it is not
  * 0, i.e., if the locale is not en_US.</td>
  * </tr>
- * 
  * <tr>
  * <td>charset_database_number</td>
  * <td>Q_CHARSET_DATABASE_CODE == 8</td>
  * <td>2 byte integer</td>
- * 
  * <td>The value of the collation_database system variable (in the source code
  * stored in thd->variables.collation_database), which holds the code for a
  * (character set, collation) pair as described above (see Q_CHARSET_CODE).
- * 
  * collation_database was used in old versions (???WHEN). Its value was loaded
  * when issuing a "use db" query and could be changed by issuing a
  * "SET collation_database=xxx" query. It used to affect the "LOAD DATA INFILE"
- * and "CREATE TABLE" commands.
- * 
- * In newer versions, "CREATE TABLE" has been changed to take the character set
- * from the database of the created table, rather than the character set of the
- * current database. This makes a difference when creating a table in another
- * database than the current one. "LOAD DATA INFILE" has not yet changed to do
- * this, but there are plans to eventually do it, and to make collation_database
- * read-only.
- * 
- * This field is written if it is not 0.</td>
+ * and "CREATE TABLE" commands. In newer versions, "CREATE TABLE" has been
+ * changed to take the character set from the database of the created table,
+ * rather than the character set of the current database. This makes a
+ * difference when creating a table in another database than the current one.
+ * "LOAD DATA INFILE" has not yet changed to do this, but there are plans to
+ * eventually do it, and to make collation_database read-only. This field is
+ * written if it is not 0.</td>
  * </tr>
  * <tr>
  * <td>table_map_for_update</td>
  * <td>Q_TABLE_MAP_FOR_UPDATE_CODE == 9</td>
  * <td>8 byte integer</td>
- * 
  * <td>The value of the table map that is to be updated by the multi-table
  * update query statement. Every bit of this variable represents a table, and is
- * set to 1 if the corresponding table is to be updated by this statement.
- * 
- * The value of this variable is set when executing a multi-table update
- * statement and used by slave to apply filter rules without opening all the
- * tables on slave. This is required because some tables may not exist on slave
- * because of the filter rules.</td>
+ * set to 1 if the corresponding table is to be updated by this statement. The
+ * value of this variable is set when executing a multi-table update statement
+ * and used by slave to apply filter rules without opening all the tables on
+ * slave. This is required because some tables may not exist on slave because of
+ * the filter rules.</td>
  * </tr>
  * </table>
- * 
- * Query_log_event_notes_on_previous_versions Notes on Previous Versions
- * 
- * Status vars were introduced in version 5.0. To read earlier versions
- * correctly, check the length of the Post-Header.
- * 
- * The status variable Q_CATALOG_CODE == 2 existed in MySQL 5.0.x, where
- * 0<=x<=3. It was identical to Q_CATALOG_CODE, except that the string had a
- * trailing '\0'. The '\0' was removed in 5.0.4 since it was redundant (the
- * string length is stored before the string). The Q_CATALOG_CODE will never be
- * written by a new master, but can still be understood by a new slave.
- * 
- * See Q_CHARSET_DATABASE_CODE in the table above.
- * 
+ * Query_log_event_notes_on_previous_versions Notes on Previous Versions Status
+ * vars were introduced in version 5.0. To read earlier versions correctly,
+ * check the length of the Post-Header. The status variable Q_CATALOG_CODE == 2
+ * existed in MySQL 5.0.x, where 0<=x<=3. It was identical to Q_CATALOG_CODE,
+ * except that the string had a trailing '\0'. The '\0' was removed in 5.0.4
+ * since it was redundant (the string length is stored before the string). The
+ * Q_CATALOG_CODE will never be written by a new master, but can still be
+ * understood by a new slave. See Q_CHARSET_DATABASE_CODE in the table above.
  * When adding new status vars, please don't forget to update the
  * MAX_SIZE_LOG_EVENT_STATUS, and update function code_name
  * 
  * @see mysql-5.1.6/sql/logevent.cc - Query_log_event
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class QueryLogEvent extends LogEvent
-{
+public class QueryLogEvent extends LogEvent {
+
     /**
-    The maximum number of updated databases that a status of
-    Query-log-event can carry.  It can redefined within a range
-    [1.. OVER_MAX_DBS_IN_EVENT_MTS].
+     * The maximum number of updated databases that a status of Query-log-event
+     * can carry. It can redefined within a range [1..
+     * OVER_MAX_DBS_IN_EVENT_MTS].
      */
     public static final int MAX_DBS_IN_EVENT_MTS      = 16;
 
     /**
-    When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS
-    the value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the
-    mts_accessed_dbs status.
+     * When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS the
+     * value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the mts_accessed_dbs
+     * status.
      */
     public static final int OVER_MAX_DBS_IN_EVENT_MTS = 254;
 
-    
     public static final int SYSTEM_CHARSET_MBMAXLEN   = 3;
     public static final int NAME_CHAR_LEN             = 64;
     /* Field/table name length */
@@ -340,23 +281,65 @@ public class QueryLogEvent extends LogEvent
      * log_event status vars estimation:
      */
     public static final int MAX_SIZE_LOG_EVENT_STATUS = (1 + 4 /* type, flags2 */
-            + 1 + 8 /* type, sql_mode */
-            + 1 + 1 + 255 /* type, length, catalog */
-            + 1 + 4 /* type, auto_increment */
-            + 1 + 6 /* type, charset */
-            + 1 + 1 + 255 /* type, length, time_zone */
-            + 1 + 2 /* type, lc_time_names_number */
-            + 1 + 2 /* type, charset_database_number */
-            + 1 + 8 /* type, table_map_for_update */
-            + 1 + 4 /* type, master_data_written */
-            /* type, db_1, db_2, ... */ 
+                                                         + 1 + 8 /*
+                                                                  * type,
+                                                                  * sql_mode
+                                                                  */
+                                                         + 1 + 1 + 255 /*
+                                                                        * type,
+                                                                        * length
+                                                                        * ,
+                                                                        * catalog
+                                                                        */
+                                                         + 1 + 4 /*
+                                                                  * type,
+                                                                  * auto_increment
+                                                                  */
+                                                         + 1 + 6 /*
+                                                                  * type,
+                                                                  * charset
+                                                                  */
+                                                         + 1 + 1 + 255 /*
+                                                                        * type,
+                                                                        * length
+                                                                        * ,
+                                                                        * time_zone
+                                                                        */
+                                                         + 1 + 2 /*
+                                                                  * type,
+                                                                  * lc_time_names_number
+                                                                  */
+                                                         + 1 + 2 /*
+                                                                  * type,
+                                                                  * charset_database_number
+                                                                  */
+                                                         + 1 + 8 /*
+                                                                  * type,
+                                                                  * table_map_for_update
+                                                                  */
+                                                         + 1 + 4 /*
+                                                                  * type,
+                                                                  * master_data_written
+                                                                  */
+                                                         /*
+                                                          * type, db_1, db_2,
+                                                          * ...
+                                                          */
                                                          /* type, microseconds */
-            /* MariaDb type, sec_part of NOW() */ 
-            + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3
-            + 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */);
+                                                         /*
+                                                          * MariaDb type,
+                                                          * sec_part of NOW()
+                                                          */
+                                                         + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 32 * 3
+                                                         + 1 + 60/*
+                                                                  * type ,
+                                                                  * user_len ,
+                                                                  * user ,
+                                                                  * host_len ,
+                                                                  * host
+                                                                  */);
     /**
      * Fixed data part:
-     * 
      * <ul>
      * <li>4 bytes. The ID of the thread that issued this statement. Needed for
      * temporary tables. This is also useful for a DBA for knowing who did what
@@ -385,7 +368,6 @@ public class QueryLogEvent extends LogEvent
      * <li>2 bytes (not present in v1, v3). The length of the status variable
      * block.</li>
      * </ul>
-     * 
      * Variable part:
      * <ul>
      * <li>Zero or more status variables (not present in v1, v3). Each status
@@ -397,7 +379,6 @@ public class QueryLogEvent extends LogEvent
      * the variable part (the sizes are given in the fixed data part), so by
      * subtraction it can know the size of the statement.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private String          user;
@@ -409,7 +390,8 @@ public class QueryLogEvent extends LogEvent
     protected final String  dbname;
 
     /** The number of seconds the query took to run on the master. */
-    //  The time in seconds that the statement took to execute. Only useful for inspection by the DBA
+    // The time in seconds that the statement took to execute. Only useful for
+    // inspection by the DBA
     private final long      execTime;
     private final int       errorCode;
     private final long      sessionId;                                                            /* thread_id */
@@ -434,9 +416,8 @@ public class QueryLogEvent extends LogEvent
 
     private String          timezone;
 
-    public QueryLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent) throws IOException
-    {
+    public QueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
+                                                                                                        throws IOException{
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -447,8 +428,7 @@ public class QueryLogEvent extends LogEvent
          * format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same
          * for 3.23, 4.0 & 5.0.
          */
-        if (buffer.limit() < (commonHeaderLen + postHeaderLen))
-        {
+        if (buffer.limit() < (commonHeaderLen + postHeaderLen)) {
             throw new IOException("Query event length is too short.");
         }
         int dataLen = buffer.limit() - (commonHeaderLen + postHeaderLen);
@@ -462,24 +442,21 @@ public class QueryLogEvent extends LogEvent
         errorCode = buffer.getUint16(); // Q_ERR_CODE_OFFSET
 
         /*
-         * 5.0 format starts here. Depending on the format, we may or not
-         * have affected/warnings etc The remaining post-header to be parsed
-         * has length:
+         * 5.0 format starts here. Depending on the format, we may or not have
+         * affected/warnings etc The remaining post-header to be parsed has
+         * length:
          */
         int statusVarsLen = 0;
-        if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN)
-        {
+        if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN) {
             statusVarsLen = buffer.getUint16(); // Q_STATUS_VARS_LEN_OFFSET
             /*
-              Check if status variable length is corrupt and will lead to very
-              wrong data. We could be even more strict and require data_len to
-              be even bigger, but this will suffice to catch most corruption
-              errors that can lead to a crash.
+             * Check if status variable length is corrupt and will lead to very
+             * wrong data. We could be even more strict and require data_len to
+             * be even bigger, but this will suffice to catch most corruption
+             * errors that can lead to a crash.
              */
-            if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS))
-            {
-                throw new IOException("status_vars_len (" + statusVarsLen
-                        + ") > data_len (" + dataLen + ")");
+            if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS)) {
+                throw new IOException("status_vars_len (" + statusVarsLen + ") > data_len (" + dataLen + ")");
             }
             dataLen -= statusVarsLen;
         }
@@ -501,27 +478,19 @@ public class QueryLogEvent extends LogEvent
         /* A 2nd variable part; this is common to all versions */
         final int queryLen = dataLen - dbLen - 1;
         dbname = buffer.getFixString(dbLen + 1);
-        if (clientCharset >= 0)
-        {
+        if (clientCharset >= 0) {
             charsetName = CharsetConversion.getJavaCharset(clientCharset);
 
-            if ((charsetName != null) && (Charset.isSupported(charsetName)))
-            {
+            if ((charsetName != null) && (Charset.isSupported(charsetName))) {
                 query = buffer.getFixString(queryLen, charsetName);
-            }
-            else
-            {
-                logger.warn("unsupported character set in query log: "
-                        + "\n    ID = " + clientCharset + ", Charset = "
-                        + CharsetConversion.getCharset(clientCharset)
-                        + ", Collation = "
+            } else {
+                logger.warn("unsupported character set in query log: " + "\n    ID = " + clientCharset + ", Charset = "
+                            + CharsetConversion.getCharset(clientCharset) + ", Collation = "
                             + CharsetConversion.getCollation(clientCharset));
 
                 query = buffer.getFixString(queryLen);
             }
-        }
-        else
-        {
+        } else {
             query = buffer.getFixString(queryLen);
         }
     }
@@ -569,34 +538,46 @@ public class QueryLogEvent extends LogEvent
     public static final int Q_INVOKER                         = 11;
 
     /**
-        Q_UPDATED_DB_NAMES status variable collects of the updated databases
-        total number and their names to be propagated to the slave in order
-        to facilitate the parallel applying of the Query events.
+     * Q_UPDATED_DB_NAMES status variable collects of the updated databases
+     * total number and their names to be propagated to the slave in order to
+     * facilitate the parallel applying of the Query events.
      */
     public static final int Q_UPDATED_DB_NAMES                = 12;
 
     public static final int Q_MICROSECONDS                    = 13;
+    /**
+     * A old (unused now) code for Query_log_event status similar to
+     * G_COMMIT_TS.
+     */
+    public static final int Q_COMMIT_TS                       = 14;
+    /**
+     * A code for Query_log_event status, similar to G_COMMIT_TS2.
+     */
+    public static final int Q_COMMIT_TS2                      = 15;
+    /**
+     * The master connection @@session.explicit_defaults_for_timestamp which is
+     * recorded for queries, CREATE and ALTER table that is defined with a
+     * TIMESTAMP column, that are dependent on that feature. For pre-WL6292
+     * master's the associated with this code value is zero.
+     */
+    public static final int Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP = 16;
 
     /**
      * FROM MariaDB 5.5.34
      */
     public static final int Q_HRNOW                           = 128;
 
-    private final void unpackVariables(LogBuffer buffer, final int end)
-            throws IOException
-    {
+    private final void unpackVariables(LogBuffer buffer, final int end) throws IOException {
         int code = -1;
-        try
-        {
-            while (buffer.position() < end)
-            {
-                switch (code = buffer.getUint8())
-                {
+        try {
+            while (buffer.position() < end) {
+                switch (code = buffer.getUint8()) {
                     case Q_FLAGS2_CODE:
                         flags2 = buffer.getUint32();
                         break;
                     case Q_SQL_MODE_CODE:
-                    sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode is ulonglong
+                        sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode
+                                                       // is ulonglong
                         break;
                     case Q_CATALOG_NZ_CODE:
                         catalog = buffer.getString();
@@ -634,7 +615,8 @@ public class QueryLogEvent extends LogEvent
                         buffer.forward(8);
                         break;
                     case Q_MASTER_DATA_WRITTEN_CODE:
-                    // data_written = master_data_written = buffer.getUint32();
+                        // data_written = master_data_written =
+                        // buffer.getUint32();
                         buffer.forward(4);
                         break;
                     case Q_INVOKER:
@@ -648,9 +630,10 @@ public class QueryLogEvent extends LogEvent
                     case Q_UPDATED_DB_NAMES:
                         int mtsAccessedDbs = buffer.getUint8();
                         /**
-                        Notice, the following check is positive also in case of
-                        the master's MAX_DBS_IN_EVENT_MTS > the slave's one and the event 
-                        contains e.g the master's MAX_DBS_IN_EVENT_MTS db:s.
+                         * Notice, the following check is positive also in case
+                         * of the master's MAX_DBS_IN_EVENT_MTS > the slave's
+                         * one and the event contains e.g the master's
+                         * MAX_DBS_IN_EVENT_MTS db:s.
                          */
                         if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) {
                             mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS;
@@ -662,30 +645,30 @@ public class QueryLogEvent extends LogEvent
                             mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
                         }
                         break;
+                    case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
+                        buffer.forward(1);
+                        break;
                     case Q_HRNOW:
                         // int when_sec_part = buffer.getUint24();
                         buffer.forward(3);
                         break;
                     default:
-                    /* That's why you must write status vars in growing order of code */
-                    if (logger.isDebugEnabled())
-                        logger.debug("Query_log_event has unknown status vars (first has code: "
+                        /*
+                         * That's why you must write status vars in growing
+                         * order of code
+                         */
+                        if (logger.isDebugEnabled()) logger.debug("Query_log_event has unknown status vars (first has code: "
                                                                   + code + "), skipping the rest of them");
                         break; // Break loop
                 }
             }
-        }
-        catch (RuntimeException e)
-        {
-            throw new IOException("Read " + findCodeName(code) + " error: "
-                    + e.getMessage(), e);
+        } catch (RuntimeException e) {
+            throw new IOException("Read " + findCodeName(code) + " error: " + e.getMessage(), e);
         }
     }
 
-    private static final String findCodeName(final int code)
-    {
-        switch (code)
-        {
+    private static final String findCodeName(final int code) {
+        switch (code) {
             case Q_FLAGS2_CODE:
                 return "Q_FLAGS2_CODE";
             case Q_SQL_MODE_CODE:
@@ -716,66 +699,54 @@ public class QueryLogEvent extends LogEvent
         return "CODE#" + code;
     }
 
-    public final String getUser()
-    {
+    public final String getUser() {
         return user;
     }
 
-    public final String getHost()
-    {
+    public final String getHost() {
         return host;
     }
 
-    public final String getQuery()
-    {
+    public final String getQuery() {
         return query;
     }
 
-    public final String getCatalog()
-    {
+    public final String getCatalog() {
         return catalog;
     }
 
-    public final String getDbName()
-    {
+    public final String getDbName() {
         return dbname;
     }
 
     /**
      * The number of seconds the query took to run on the master.
      */
-    public final long getExecTime()
-    {
+    public final long getExecTime() {
         return execTime;
     }
 
-    public final int getErrorCode()
-    {
+    public final int getErrorCode() {
         return errorCode;
     }
 
-    public final long getSessionId()
-    {
+    public final long getSessionId() {
         return sessionId;
     }
 
-    public final long getAutoIncrementIncrement()
-    {
+    public final long getAutoIncrementIncrement() {
         return autoIncrementIncrement;
     }
 
-    public final long getAutoIncrementOffset()
-    {
+    public final long getAutoIncrementOffset() {
         return autoIncrementOffset;
     }
 
-    public final String getCharsetName()
-    {
+    public final String getCharsetName() {
         return charsetName;
     }
 
-    public final String getTimezone()
-    {
+    public final String getTimezone() {
         return timezone;
     }
 
@@ -784,8 +755,7 @@ public class QueryLogEvent extends LogEvent
      * 
      * @return Returns the charsetID.
      */
-    public final int getClientCharset()
-    {
+    public final int getClientCharset() {
         return clientCharset;
     }
 
@@ -794,8 +764,7 @@ public class QueryLogEvent extends LogEvent
      * 
      * @return Returns the clientCollationId.
      */
-    public final int getClientCollation()
-    {
+    public final int getClientCollation() {
         return clientCollation;
     }
 
@@ -804,19 +773,16 @@ public class QueryLogEvent extends LogEvent
      * 
      * @return Returns the serverCollationId.
      */
-    public final int getServerCollation()
-    {
+    public final int getServerCollation() {
         return serverCollation;
     }
 
     /**
      * Returns the sql_mode value.
-     * 
      * <p>
      * The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
      * and see mysql_priv.h for a list of the possible flags. Currently
      * (2007-10-04), the following flags are available:
-     * 
      * <ul>
      * <li>MODE_REAL_AS_FLOAT==0x1</li>
      * <li>MODE_PIPES_AS_CONCAT==0x2</li>
@@ -851,14 +817,12 @@ public class QueryLogEvent extends LogEvent
      * <li>MODE_NO_ENGINE_SUBSTITUTION=0x40000000</li>
      * <li>MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000</li>
      * </ul>
-     * 
      * All these flags are replicated from the server. However, all flags except
      * MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always
      * preserves its old value of MODE_NO_DIR_IN_CREATE. This field is always
      * written to the binlog.
      */
-    public final long getSqlMode()
-    {
+    public final long getSqlMode() {
         return sql_mode;
     }
 
@@ -880,40 +844,35 @@ public class QueryLogEvent extends LogEvent
      * Manual. This field is always written to the binlog in version >= 5.0, and
      * never written in version < 5.0.
      */
-    public final long getFlags2()
-    {
+    public final long getFlags2() {
         return flags2;
     }
 
     /**
      * Returns the OPTION_AUTO_IS_NULL flag.
      */
-    public final boolean isAutoIsNull()
-    {
+    public final boolean isAutoIsNull() {
         return ((flags2 & OPTION_AUTO_IS_NULL) == OPTION_AUTO_IS_NULL);
     }
 
     /**
      * Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
      */
-    public final boolean isForeignKeyChecks()
-    {
+    public final boolean isForeignKeyChecks() {
         return ((flags2 & OPTION_NO_FOREIGN_KEY_CHECKS) != OPTION_NO_FOREIGN_KEY_CHECKS);
     }
 
     /**
      * Returns the OPTION_NOT_AUTOCOMMIT flag.
      */
-    public final boolean isAutocommit()
-    {
+    public final boolean isAutocommit() {
         return ((flags2 & OPTION_NOT_AUTOCOMMIT) != OPTION_NOT_AUTOCOMMIT);
     }
 
     /**
      * Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
      */
-    public final boolean isUniqueChecks()
-    {
+    public final boolean isUniqueChecks() {
         return ((flags2 & OPTION_RELAXED_UNIQUE_CHECKS) != OPTION_RELAXED_UNIQUE_CHECKS);
     }
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java

@@ -7,30 +7,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1
  * does not need it (it's repeatable again) so this event needn't be written in
  * 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it
- * does not cause bugs).
- * 
- * The state of the random number generation consists of 128 bits, which are
- * stored internally as two 64-bit numbers.
- * 
- * Binary Format
- * 
+ * does not cause bugs). The state of the random number generation consists of
+ * 128 bits, which are stored internally as two 64-bit numbers. Binary Format
  * The Post-Header for this event type is empty. The Body has two components:
- * 
  * <table>
  * <caption>Body for Rand_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>seed1</td>
  * <td>8 byte unsigned integer</td>
  * <td>64 bit random seed1.</td>
  * </tr>
- * 
  * <tr>
  * <td>seed2</td>
  * <td>8 byte unsigned integer</td>
@@ -41,19 +32,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class RandLogEvent extends LogEvent
-{
+public final class RandLogEvent extends LogEvent {
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>8 bytes. The value for the first seed.</li>
      * <li>8 bytes. The value for the second seed.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long      seed1;
@@ -63,21 +51,17 @@ public final class RandLogEvent extends LogEvent
     public static final int RAND_SEED1_OFFSET = 0;
     public static final int RAND_SEED2_OFFSET = 8;
 
-    public RandLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public RandLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         /* The Post-Header is empty. The Variable Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
-                + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
                         + RAND_SEED1_OFFSET);
         seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET);
         seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET);
     }
 
-    public final String getQuery()
-    {
+    public final String getQuery() {
         return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java

@@ -4,57 +4,44 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * This will be deprecated when we move to using sequence ids.
- * 
- * Binary Format
- * 
- * The Post-Header has one component:
- * 
+ * This will be deprecated when we move to using sequence ids. Binary Format The
+ * Post-Header has one component:
  * <table>
  * <caption>Post-Header for Rotate_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>position</td>
  * <td>8 byte integer</td>
  * <td>The position within the binlog to rotate to.</td>
  * </tr>
- * 
  * </table>
- * 
  * The Body has one component:
- * 
  * <table>
  * <caption>Body for Rotate_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>new_log</td>
  * <td>variable length string without trailing zero, extending to the end of the
  * event (determined by the length field of the Common-Header)</td>
  * <td>Name of the binlog to rotate to.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class RotateLogEvent extends LogEvent
-{
+public final class RotateLogEvent extends LogEvent {
+
     /**
      * Fixed data part:
-     * 
      * <ul>
      * <li>8 bytes. The position of the first event in the next log file. Always
      * contains the number 4 (meaning the next event starts at position 4 in the
@@ -62,14 +49,11 @@ public final class RotateLogEvent extends LogEvent
      * is assumed to be 4.</li>
      * </ul>
      * <p>
-     * 
      * Variable data part:
-     * 
      * <ul>
      * <li>The name of the next binary log. The filename is not null-terminated.
      * Its length is the event size minus the size of the fixed parts.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final String          filename;
@@ -91,22 +75,22 @@ public final class RotateLogEvent extends LogEvent
      * 
      * @throws MySQLExtractException
      */
-    public RotateLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public RotateLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int headerSize = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1];
 
         buffer.position(headerSize + R_POS_OFFSET);
-        position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET)
+        position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf
+                                                                  // +
+                                                                  // R_POS_OFFSET)
 
         final int filenameOffset = headerSize + postHeaderLen;
         int filenameLen = buffer.limit() - filenameOffset;
-        if (filenameLen > FN_REFLEN - 1)
-            filenameLen = FN_REFLEN - 1;
+        if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1;
         buffer.position(filenameOffset);
+
         filename = buffer.getFixString(filenameLen);
     }
 
@@ -114,8 +98,7 @@ public final class RotateLogEvent extends LogEvent
      * Creates a new <code>Rotate_log_event</code> without log information. This
      * is used to generate missing log rotation events.
      */
-    public RotateLogEvent(String filename)
-    {
+    public RotateLogEvent(String filename){
         super(ROTATE_HEADER);
 
         this.filename = filename;
@@ -125,21 +108,18 @@ public final class RotateLogEvent extends LogEvent
     /**
      * Creates a new <code>Rotate_log_event</code> without log information.
      */
-    public RotateLogEvent(String filename, final long position)
-    {
+    public RotateLogEvent(String filename, final long position){
         super(ROTATE_HEADER);
 
         this.filename = filename;
         this.position = position;
     }
 
-    public final String getFilename()
-    {
+    public final String getFilename() {
         return filename;
     }
 
-    public final long getPosition()
-    {
+    public final long getPosition() {
         return position;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java

@@ -8,6 +8,8 @@ import java.util.BitSet;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.taobao.tddl.dbsync.binlog.JsonConversion;
+import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
@@ -24,6 +26,8 @@ public final class RowsLogBuffer {
 
     public static final long   DATETIMEF_INT_OFS = 0x8000000000L;
     public static final long   TIMEF_INT_OFS     = 0x800000L;
+    public static final long   TIMEF_OFS         = 0x800000000000L;
+
     private final LogBuffer    buffer;
     private final int          columnLen;
     private final String       charsetName;
@@ -162,10 +166,13 @@ public final class RowsLogBuffer {
 
             case LogEvent.MYSQL_TYPE_TIMESTAMP:
             case LogEvent.MYSQL_TYPE_DATETIME:
+            case LogEvent.MYSQL_TYPE_TIMESTAMP2:
+            case LogEvent.MYSQL_TYPE_DATETIME2:
                 javaType = Types.TIMESTAMP;
                 break;
 
             case LogEvent.MYSQL_TYPE_TIME:
+            case LogEvent.MYSQL_TYPE_TIME2:
                 javaType = Types.TIME;
                 break;
 
@@ -437,14 +444,23 @@ public final class RowsLogBuffer {
                         break;
                 }
 
+                String second = null;
                 if (tv_sec == 0) {
-                    value = "0000-00-00 00:00:00";
+                    second = "0000-00-00 00:00:00";
                 } else {
                     Timestamp time = new Timestamp(tv_sec * 1000);
-                    time.setNanos(tv_usec * 1000);
-                    String v = time.toString();
-                    value = v.substring(0, v.length() - 2);
+                    second = time.toString();
+                    second = second.substring(0, second.length() - 2);// 去掉毫秒精度.0
                 }
+
+                if (meta >= 1) {
+                    String microSecond = usecondsToStr(tv_usec, meta);
+                    microSecond = microSecond.substring(0, meta);
+                    value = second + '.' + microSecond;
+                } else {
+                    value = second;
+                }
+
                 javaType = Types.TIMESTAMP;
                 length = 4 + (meta + 1) / 2;
                 break;
@@ -487,7 +503,6 @@ public final class RowsLogBuffer {
                  * .YYdddddh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff
                  */
                 long intpart = buffer.getBeUlong40() - DATETIMEF_INT_OFS; // big-endian
-                @SuppressWarnings("unused")
                 int frac = 0;
                 switch (meta) {
                     case 0:
@@ -510,8 +525,9 @@ public final class RowsLogBuffer {
                         break;
                 }
 
+                String second = null;
                 if (intpart == 0) {
-                    value = "0000-00-00 00:00:00";
+                    second = "0000-00-00 00:00:00";
                 } else {
                     // 构造TimeStamp只处理到秒
                     long ymd = intpart >> 17;
@@ -524,7 +540,7 @@ public final class RowsLogBuffer {
                     // % (1 << 5)), (int) (hms >> 12),
                     // (int) ((hms >> 6) % (1 << 6)), (int) (hms % (1 << 6)));
                     // value = new Timestamp(cal.getTimeInMillis());
-                    value = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                    second = String.format("%04d-%02d-%02d %02d:%02d:%02d",
                         (int) (ym / 13),
                         (int) (ym % 13),
                         (int) (ymd % (1 << 5)),
@@ -532,6 +548,15 @@ public final class RowsLogBuffer {
                         (int) ((hms >> 6) % (1 << 6)),
                         (int) (hms % (1 << 6)));
                 }
+
+                if (meta >= 1) {
+                    String microSecond = usecondsToStr(frac, meta);
+                    microSecond = microSecond.substring(0, meta);
+                    value = second + '.' + microSecond;
+                } else {
+                    value = second;
+                }
+
                 javaType = Types.TIMESTAMP;
                 length = 5 + (meta + 1) / 2;
                 break;
@@ -603,7 +628,8 @@ public final class RowsLogBuffer {
                             frac -= 0x100; /* -(0x100 - frac) */
                             // fraclong = frac * 10000;
                         }
-                        ltime = intpart << 24 + frac * 10000;
+                        frac = frac * 10000;
+                        ltime = intpart << 24;
                         break;
                     case 3:
                     case 4:
@@ -619,12 +645,14 @@ public final class RowsLogBuffer {
                             frac -= 0x10000; /* -(0x10000-frac) */
                             // fraclong = frac * 100;
                         }
-                        ltime = intpart << 24 + frac * 100;
+                        frac = frac * 100;
+                        ltime = intpart << 24;
                         break;
                     case 5:
                     case 6:
-                        intpart = buffer.getBeUlong48() - TIMEF_INT_OFS;
+                        intpart = buffer.getBeUlong48() - TIMEF_OFS;
                         ltime = intpart;
+                        frac = (int) (intpart % (1L << 24));
                         break;
                     default:
                         intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
@@ -632,8 +660,9 @@ public final class RowsLogBuffer {
                         break;
                 }
 
+                String second = null;
                 if (intpart == 0) {
-                    value = "00:00:00";
+                    second = "00:00:00";
                 } else {
                     // 目前只记录秒，不处理us frac
                     // if (cal == null) cal = Calendar.getInstance();
@@ -644,13 +673,21 @@ public final class RowsLogBuffer {
                     // value = new Time(cal.getTimeInMillis());
                     long ultime = Math.abs(ltime);
                     intpart = ultime >> 24;
-                    value = String.format("%s%02d:%02d:%02d",
+                    second = String.format("%s%02d:%02d:%02d",
                         ltime >= 0 ? "" : "-",
                         (int) ((intpart >> 12) % (1 << 10)),
                         (int) ((intpart >> 6) % (1 << 6)),
                         (int) (intpart % (1 << 6)));
                 }
 
+                if (meta >= 1) {
+                    String microSecond = usecondsToStr(Math.abs(frac), meta);
+                    microSecond = microSecond.substring(0, meta);
+                    value = second + '.' + microSecond;
+                } else {
+                    value = second;
+                }
+
                 javaType = Types.TIME;
                 length = 3 + (meta + 1) / 2;
                 break;
@@ -911,6 +948,42 @@ public final class RowsLogBuffer {
                 length = len;
                 break;
             }
+            case LogEvent.MYSQL_TYPE_JSON: {
+                switch (meta) {
+                    case 1: {
+                        len = buffer.getUint8();
+                        break;
+                    }
+                    case 2: {
+                        len = buffer.getUint16();
+                        break;
+                    }
+                    case 3: {
+                        len = buffer.getUint24();
+                        break;
+                    }
+                    case 4: {
+                        len = (int) buffer.getUint32();
+                        break;
+                    }
+                    default:
+                        throw new IllegalArgumentException("!! Unknown JSON packlen = " + meta);
+                }
+                // len = buffer.getUint16();
+                // buffer.forward(meta - 4);
+                int position = buffer.position();
+                Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName);
+                StringBuilder builder = new StringBuilder();
+                jsonValue.toJsonString(builder, charsetName);
+                value = builder.toString();
+                buffer.position(position + len);
+                // byte[] binary = new byte[len];
+                // buffer.fillBytes(binary, 0, len);
+                // value = binary;
+                javaType = Types.VARCHAR;
+                length = len;
+                break;
+            }
             case LogEvent.MYSQL_TYPE_GEOMETRY: {
                 /*
                  * MYSQL_TYPE_GEOMETRY: copy from BLOB or TEXT
@@ -973,4 +1046,24 @@ public final class RowsLogBuffer {
     public final int getLength() {
         return length;
     }
+
+    private String usecondsToStr(int frac, int meta) {
+        String sec = String.valueOf(frac);
+        if (meta > 6) {
+            throw new IllegalArgumentException("unknow useconds meta : " + meta);
+        }
+
+        if (sec.length() < 6) {
+            StringBuilder result = new StringBuilder(6);
+            int len = 6 - sec.length();
+            for (; len > 0; len--) {
+                result.append('0');
+            }
+            result.append(sec);
+            sec = result.toString();
+        }
+
+        return sec.substring(0, meta);
+    }
+
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java

@@ -12,41 +12,33 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class RowsLogEvent extends LogEvent
-{
+public abstract class RowsLogEvent extends LogEvent {
+
     /**
      * Fixed data part:
-     * 
      * <ul>
      * <li>6 bytes. The table ID.</li>
      * <li>2 bytes. Reserved for future use.</li>
      * </ul>
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>Packed integer. The number of columns in the table.</li>
-     * 
      * <li>Variable-sized. Bit-field indicating whether each column is used, one
      * bit per column. For this field, the amount of storage required for N
      * columns is INT((N+7)/8) bytes.</li>
-     * 
      * <li>Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating
      * whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one
      * bit per column. For this field, the amount of storage required for N
      * columns is INT((N+7)/8) bytes.</li>
-     * 
      * <li>Variable-sized. A sequence of zero or more rows. The end is
      * determined by the size of the event. Each row has the following format:
-     * 
      * <ul>
      * <li>Variable-sized. Bit-field indicating whether each field in the row is
      * NULL. Only columns that are "used" according to the second field in the
      * variable data part are listed here. If the second field in the variable
      * data part has N one-bits, the amount of storage required for this field
      * is INT((N+7)/8) bytes.</li>
-     * 
      * <li>Variable-sized. The row-image, containing values of all table fields.
      * This only lists table fields that are used (according to the second field
      * of the variable data part) and non-NULL (according to the previous
@@ -54,16 +46,18 @@ public abstract class RowsLogEvent extends LogEvent
      * number of zero bits in the previous field (not counting padding bits in
      * the last byte). The format of each value is described in the
      * log_event_print_value() function in log_event.cc.</li>
-     * 
      * <li>(for UPDATE_ROWS_EVENT only) the previous two fields are repeated,
      * representing a second table row.</li>
      * </ul>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long       tableId;                           /* Table ID */
-    private TableMapLogEvent table;                             /* The table the rows belong to */
+    private TableMapLogEvent table;                             /*
+                                                                  * The table
+                                                                  * the rows
+                                                                  * belong to
+                                                                  */
 
     /** Bitmap denoting columns available */
     protected final int      columnLen;
@@ -78,15 +72,17 @@ public abstract class RowsLogEvent extends LogEvent
     protected final BitSet   changeColumns;
 
     /** XXX: Don't handle buffer in another thread. */
-    private final LogBuffer  rowsBuf;                           /* The rows in packed format */
+    private final LogBuffer  rowsBuf;                           /*
+                                                                  * The rows in
+                                                                  * packed
+                                                                  * format
+                                                                  */
 
     /**
-     * enum enum_flag
-     * 
-     * These definitions allow you to combine the flags into an appropriate flag
-     * set using the normal bitwise operators. The implicit conversion from an
-     * enum-constant to an integer is accepted by the compiler, which is then
-     * used to set the real set of flags.
+     * enum enum_flag These definitions allow you to combine the flags into an
+     * appropriate flag set using the normal bitwise operators. The implicit
+     * conversion from an enum-constant to an integer is accepted by the
+     * compiler, which is then used to set the real set of flags.
      */
     private final int        flags;
 
@@ -112,29 +108,25 @@ public abstract class RowsLogEvent extends LogEvent
     public static final int  RW_V_TAG_LEN            = 1;
     public static final int  RW_V_EXTRAINFO_TAG      = 0;
 
-
-    public RowsLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
         int headerLen = 0;
         buffer.position(commonHeaderLen + RW_MAPID_OFFSET);
-        if (postHeaderLen == 6)
-        {
-            /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
+        if (postHeaderLen == 6) {
+            /*
+             * Master is of an intermediate source tree before 5.1.4. Id is 4
+             * bytes
+             */
             tableId = buffer.getUint32();
-        }
-        else
-        {
+        } else {
             tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET
         }
         flags = buffer.getUint16();
 
-        if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2)
-        {
+        if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2) {
             headerLen = buffer.getUint16();
             headerLen -= 2;
             int start = buffer.position();
@@ -148,7 +140,8 @@ public abstract class RowsLogEvent extends LogEvent
                         int val = checkLen - EXTRA_ROW_INFO_HDR_BYTES;
                         assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_FORMAT_OFFSET
                         for (int j = 0; j < val; j++) {
-                          assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i
+                            assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES
+                                                               // + i
                         }
                         break;
                     default:
@@ -162,12 +155,9 @@ public abstract class RowsLogEvent extends LogEvent
         columnLen = (int) buffer.getPackedLong();
         columns = buffer.getBitmap(columnLen);
 
-        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT)
-        {
+        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) {
             changeColumns = buffer.getBitmap(columnLen);
-        }
-        else
-        {
+        } else {
             changeColumns = columns;
         }
 
@@ -176,46 +166,38 @@ public abstract class RowsLogEvent extends LogEvent
         rowsBuf = buffer.duplicate(dataSize);
     }
 
-    public final void fillTable(LogContext context)
-    {
+    public final void fillTable(LogContext context) {
         table = context.getTable(tableId);
 
         // end of statement check:
-        if ((flags & RowsLogEvent.STMT_END_F) != 0)
-        {
+        if ((flags & RowsLogEvent.STMT_END_F) != 0) {
             // Now is safe to clear ignored map (clear_tables will also
             // delete original table map events stored in the map).
             context.clearAllTables();
         }
     }
 
-    public final long getTableId()
-    {
+    public final long getTableId() {
         return tableId;
     }
 
-    public final TableMapLogEvent getTable()
-    {
+    public final TableMapLogEvent getTable() {
         return table;
     }
 
-    public final BitSet getColumns()
-    {
+    public final BitSet getColumns() {
         return columns;
     }
 
-    public final BitSet getChangeColumns()
-    {
+    public final BitSet getChangeColumns() {
         return changeColumns;
     }
 
-    public final RowsLogBuffer getRowsBuf(String charsetName)
-    {
+    public final RowsLogBuffer getRowsBuf(String charsetName) {
         return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName);
     }
 
-    public final int getFlags(final int flags)
-    {
+    public final int getFlags(final int flags) {
         return this.flags & flags;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java

@@ -5,21 +5,18 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and
- * 4.x).
- * 
- * Format_description_log_event derives from Start_log_event_v3; it is the
+ * 4.x). Format_description_log_event derives from Start_log_event_v3; it is the
  * Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that
  * describes the other events' Common-Header/Post-Header lengths. This event is
  * sent by MySQL 5.0 whenever it starts sending a new binlog if the requested
  * position is >4 (otherwise if ==4 the event will be sent naturally).
  * 
  * @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class StartLogEventV3 extends LogEvent
-{
+public class StartLogEventV3 extends LogEvent {
+
     /**
      * We could have used SERVER_VERSION_LENGTH, but this introduces an obscure
      * dependency - if somebody decided to change SERVER_VERSION_LENGTH this
@@ -34,9 +31,7 @@ public class StartLogEventV3 extends LogEvent
     protected int           binlogVersion;
     protected String        serverVersion;
 
-    public StartLogEventV3(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public StartLogEventV3(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         buffer.position(descriptionEvent.commonHeaderLen);
@@ -44,18 +39,15 @@ public class StartLogEventV3 extends LogEvent
         serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
     }
 
-    public StartLogEventV3()
-    {
+    public StartLogEventV3(){
         super(new LogHeader(START_EVENT_V3));
     }
 
-    public final String getServerVersion()
-    {
+    public final String getServerVersion() {
         return serverVersion;
     }
 
-    public final int getBinlogVersion()
-    {
+    public final int getBinlogVersion() {
         return binlogVersion;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java

@@ -4,19 +4,15 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * Stop_log_event.
- * 
- * The Post-Header and Body for this event type are empty; it only has the
- * Common-Header.
+ * Stop_log_event. The Post-Header and Body for this event type are empty; it
+ * only has the Common-Header.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class StopLogEvent extends LogEvent
-{
-    public StopLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent description_event)
-    {
+public final class StopLogEvent extends LogEvent {
+
+    public StopLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent description_event){
         super(header);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java

@@ -8,44 +8,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * In row-based mode, every row operation event is preceded by a
  * Table_map_log_event which maps a table definition to a number. The table
- * definition consists of database name, table name, and column definitions.
- * 
- * The Post-Header has the following components:
- * 
+ * definition consists of database name, table name, and column definitions. The
+ * Post-Header has the following components:
  * <table>
  * <caption>Post-Header for Table_map_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>table_id</td>
  * <td>6 bytes unsigned integer</td>
  * <td>The number that identifies the table.</td>
  * </tr>
- * 
  * <tr>
  * <td>flags</td>
  * <td>2 byte bitfield</td>
  * <td>Reserved for future use; currently always 0.</td>
  * </tr>
- * 
  * </table>
- * 
  * The Body has the following components:
- * 
  * <table>
  * <caption>Body for Table_map_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>database_name</td>
  * <td>one byte string length, followed by null-terminated string</td>
@@ -55,20 +45,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * by a terminating 0 byte. (Note the redundancy in the representation of the
  * length.)</td>
  * </tr>
- * 
  * <tr>
  * <td>table_name</td>
  * <td>one byte string length, followed by null-terminated string</td>
  * <td>The name of the table, encoded the same way as the database name above.</td>
  * </tr>
- * 
  * <tr>
  * <td>column_count</td>
  * <td>packed_integer "Packed Integer"</td>
  * <td>The number of columns in the table, represented as a packed
  * variable-length integer.</td>
  * </tr>
- * 
  * <tr>
  * <td>column_type</td>
  * <td>List of column_count 1 byte enumeration values</td>
@@ -78,13 +65,11 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * listed in the table Table_table_map_log_event_column_types "below" (along
  * with description of the associated metadata field).</td>
  * </tr>
- * 
  * <tr>
  * <td>metadata_length</td>
  * <td>packed_integer "Packed Integer"</td>
  * <td>The length of the following metadata block</td>
  * </tr>
- * 
  * <tr>
  * <td>metadata</td>
  * <td>list of metadata for each column</td>
@@ -93,7 +78,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * metadata for each column are listed in the table
  * Table_table_map_log_event_column_types "below".</td>
  * </tr>
- * 
  * <tr>
  * <td>null_bits</td>
  * <td>column_count bits, rounded up to nearest byte</td>
@@ -104,13 +88,10 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * first byte, the ninth is in the least significant bit of the second byte, and
  * so on.</td>
  * </tr>
- * 
  * </table>
- * 
  * The table below lists all column types, along with the numerical identifier
  * for it and the size and interpretation of meta-data used to describe the
  * type.
- * 
  * <table>
  * <caption>Table_map_log_event column types: numerical identifier and
  * metadata</caption>
@@ -120,35 +101,30 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <th>Size of metadata in bytes</th>
  * <th>Description of metadata</th>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DECIMAL</td>
  * <td>0</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TINY</td>
  * <td>1</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_SHORT</td>
  * <td>2</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_LONG</td>
  * <td>3</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_FLOAT</td>
  * <td>4</td>
@@ -156,7 +132,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>1 byte unsigned integer, representing the "pack_length", which is equal
  * to sizeof(float) on the server from which the event originates.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DOUBLE</td>
  * <td>5</td>
@@ -164,63 +139,54 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>1 byte unsigned integer, representing the "pack_length", which is equal
  * to sizeof(double) on the server from which the event originates.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_NULL</td>
  * <td>6</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TIMESTAMP</td>
  * <td>7</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_LONGLONG</td>
  * <td>8</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_INT24</td>
  * <td>9</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DATE</td>
  * <td>10</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TIME</td>
  * <td>11</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DATETIME</td>
  * <td>12</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_YEAR</td>
  * <td>13</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_NEWDATE</i></td>
  * <td><i>14</i></td>
@@ -228,14 +194,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_VARCHAR</td>
  * <td>15</td>
  * <td>2 bytes</td>
  * <td>2 byte unsigned integer representing the maximum length of the string.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_BIT</td>
  * <td>16</td>
@@ -245,7 +209,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * occupied by the bitfield. The number of bytes is either int((length+7)/8) or
  * int(length/8).</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_NEWDECIMAL</td>
  * <td>246</td>
@@ -253,7 +216,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>A 1 byte unsigned int representing the precision, followed by a 1 byte
  * unsigned int representing the number of decimals.</td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_ENUM</i></td>
  * <td><i>247</i></td>
@@ -261,7 +223,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_SET</i></td>
  * <td><i>248</i></td>
@@ -269,7 +230,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TINY_BLOB</td>
  * <td>249</td>
@@ -277,7 +237,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_MEDIUM_BLOB</i></td>
  * <td><i>250</i></td>
@@ -285,7 +244,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_LONG_BLOB</i></td>
  * <td><i>251</i></td>
@@ -293,7 +251,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_BLOB</td>
  * <td>252</td>
@@ -301,7 +258,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>The pack length, i.e., the number of bytes needed to represent the length
  * of the blob: 1, 2, 3, or 4.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_VAR_STRING</td>
  * <td>253</td>
@@ -312,7 +268,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * unsigned integer representing the field size, i.e., the number of bytes
  * needed to store the length of the string.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_STRING</td>
  * <td>254</td>
@@ -321,7 +276,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * byte is the field size, i.e., the number of bytes in the representation of
  * size of the string: 3 or 4.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_GEOMETRY</td>
  * <td>255</td>
@@ -329,21 +283,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>The pack length, i.e., the number of bytes needed to represent the length
  * of the geometry: 1, 2, 3, or 4.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class TableMapLogEvent extends LogEvent
-{
+public final class TableMapLogEvent extends LogEvent {
+
     /**
      * Fixed data part:
      * <ul>
      * <li>6 bytes. The table ID.</li>
      * <li>2 bytes. Reserved for future use.</li>
      * </ul>
-     * 
      * <p>
      * Variable data part:
      * <ul>
@@ -360,7 +312,6 @@ public final class TableMapLogEvent extends LogEvent
      * one bit per column. For this field, the amount of storage required for N
      * columns is INT((N+7)/8) bytes.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     protected final String dbname;
@@ -369,14 +320,15 @@ public final class TableMapLogEvent extends LogEvent
     /**
      * Holding mysql column information.
      */
-    public static final class ColumnInfo
-    {
+    public static final class ColumnInfo {
+
         public int type;
         public int meta;
     }
 
     protected final int          columnCnt;
-    protected final ColumnInfo[] columnInfo;         // buffer for field metadata
+    protected final ColumnInfo[] columnInfo;         // buffer for field
+                                                      // metadata
 
     protected final long         tableId;
     protected BitSet             nullBits;
@@ -388,22 +340,20 @@ public final class TableMapLogEvent extends LogEvent
     /**
      * Constructor used by slave to read the event from the binary log.
      */
-    public TableMapLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public TableMapLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
         /* Read the post-header */
         buffer.position(commonHeaderLen + TM_MAPID_OFFSET);
-        if (postHeaderLen == 6)
-        {
-            /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
+        if (postHeaderLen == 6) {
+            /*
+             * Master is of an intermediate source tree before 5.1.4. Id is 4
+             * bytes
+             */
             tableId = buffer.getUint32();
-        }
-        else
-        {
+        } else {
             // DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN);
             tableId = buffer.getUlong48();
         }
@@ -419,15 +369,13 @@ public final class TableMapLogEvent extends LogEvent
         // Read column information from buffer
         columnCnt = (int) buffer.getPackedLong();
         columnInfo = new ColumnInfo[columnCnt];
-        for (int i = 0; i < columnCnt; i++)
-        {
+        for (int i = 0; i < columnCnt; i++) {
             ColumnInfo info = new ColumnInfo();
             info.type = buffer.getUint8();
             columnInfo[i] = info;
         }
 
-        if (buffer.position() < buffer.limit())
-        {
+        if (buffer.position() < buffer.limit()) {
             final int fieldSize = (int) buffer.getPackedLong();
             decodeFields(buffer, fieldSize);
             nullBits = buffer.getBitmap(columnCnt);
@@ -439,17 +387,14 @@ public final class TableMapLogEvent extends LogEvent
      * 
      * @see mysql-5.1.60/sql/rpl_utility.h
      */
-    private final void decodeFields(LogBuffer buffer, final int len)
-    {
+    private final void decodeFields(LogBuffer buffer, final int len) {
         final int limit = buffer.limit();
 
         buffer.limit(len + buffer.position());
-        for (int i = 0; i < columnCnt; i++)
-        {
+        for (int i = 0; i < columnCnt; i++) {
             ColumnInfo info = columnInfo[i];
 
-            switch (info.type)
-            {
+            switch (info.type) {
                 case MYSQL_TYPE_TINY_BLOB:
                 case MYSQL_TYPE_BLOB:
                 case MYSQL_TYPE_MEDIUM_BLOB:
@@ -457,8 +402,9 @@ public final class TableMapLogEvent extends LogEvent
                 case MYSQL_TYPE_DOUBLE:
                 case MYSQL_TYPE_FLOAT:
                 case MYSQL_TYPE_GEOMETRY:
+                case MYSQL_TYPE_JSON:
                     /*
-                  These types store a single byte.
+                     * These types store a single byte.
                      */
                     info.meta = buffer.getUint8();
                     break;
@@ -472,8 +418,7 @@ public final class TableMapLogEvent extends LogEvent
                     logger.warn("This enumeration value is only used internally "
                                 + "and cannot exist in a binlog: type=" + info.type);
                     break;
-            case MYSQL_TYPE_STRING:
-                {
+                case MYSQL_TYPE_STRING: {
                     /*
                      * log_event.h : The first byte is always
                      * MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the
@@ -494,8 +439,7 @@ public final class TableMapLogEvent extends LogEvent
                      */
                     info.meta = buffer.getUint16();
                     break;
-            case MYSQL_TYPE_NEWDECIMAL:
-                {
+                case MYSQL_TYPE_NEWDECIMAL: {
                     int x = buffer.getUint8() << 8; // precision
                     x += buffer.getUint8(); // decimals
                     info.meta = x;
@@ -503,8 +447,7 @@ public final class TableMapLogEvent extends LogEvent
                 }
                 case MYSQL_TYPE_TIME2:
                 case MYSQL_TYPE_DATETIME2:
-            case MYSQL_TYPE_TIMESTAMP2:
-                {
+                case MYSQL_TYPE_TIMESTAMP2: {
                     info.meta = buffer.getUint8();
                     break;
                 }
@@ -516,28 +459,23 @@ public final class TableMapLogEvent extends LogEvent
         buffer.limit(limit);
     }
 
-    public final String getDbName()
-    {
+    public final String getDbName() {
         return dbname;
     }
 
-    public final String getTableName()
-    {
+    public final String getTableName() {
         return tblname;
     }
 
-    public final int getColumnCnt()
-    {
+    public final int getColumnCnt() {
         return columnCnt;
     }
 
-    public final ColumnInfo[] getColumnInfo()
-    {
+    public final ColumnInfo[] getColumnInfo() {
         return columnInfo;
     }
 
-    public final long getTableId()
-    {
+    public final long getTableId() {
         return tableId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java

@@ -8,10 +8,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UnknownLogEvent extends LogEvent
-{
-    public UnknownLogEvent(LogHeader header)
-    {
+public final class UnknownLogEvent extends LogEvent {
+
+    public UnknownLogEvent(LogHeader header){
         super(header);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java

@@ -4,19 +4,16 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
 /**
  * Log row updates with a before image. The event contain several update rows
- * for a table. Note that each event contains only rows for one table.
- * 
- * Also note that the row data consists of pairs of row data: one row for the
- * old data and one row for the new data.
+ * for a table. Note that each event contains only rows for one table. Also note
+ * that the row data consists of pairs of row data: one row for the old data and
+ * one row for the new data.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UpdateRowsLogEvent extends RowsLogEvent
-{
-    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+public final class UpdateRowsLogEvent extends RowsLogEvent {
+
+    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java

@@ -8,22 +8,19 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * User_var_log_event.
- * 
- * Every time a query uses the value of a user variable, a User_var_log_event is
- * written before the Query_log_event, to set the user variable.
+ * User_var_log_event. Every time a query uses the value of a user variable, a
+ * User_var_log_event is written before the Query_log_event, to set the user
+ * variable.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UserVarLogEvent extends LogEvent
-{
+public final class UserVarLogEvent extends LogEvent {
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>4 bytes. the size of the user variable name.</li>
      * <li>The user variable name.</li>
@@ -39,7 +36,6 @@ public final class UserVarLogEvent extends LogEvent
      * <li>Variable-sized. For a string variable, this is the string. For a
      * float or integer variable, this is its value in 8 bytes.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final String       name;
@@ -66,46 +62,37 @@ public final class UserVarLogEvent extends LogEvent
     public static final int    UV_NAME_LEN_SIZE       = 4;
     public static final int    UV_CHARSET_NUMBER_SIZE = 4;
 
-    public UserVarLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent) throws IOException
-    {
+    public UserVarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
+                                                                                                          throws IOException{
         super(header);
 
         /* The Post-Header is empty. The Variable Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
-                + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
         final int nameLen = (int) buffer.getUint32();
         name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE
         isNull = (0 != buffer.getInt8());
 
-        if (isNull)
-        {
+        if (isNull) {
             type = STRING_RESULT;
             charsetNumber = 63; /* binary */
             value = null;
-        }
-        else
-        {
+        } else {
             type = buffer.getInt8(); // UV_VAL_IS_NULL
             charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE
-            final int valueLen = (int) buffer.getUint32(); // buf +  UV_CHARSET_NUMBER_SIZE
+            final int valueLen = (int) buffer.getUint32(); // buf +
+                                                           // UV_CHARSET_NUMBER_SIZE
             final int limit = buffer.limit(); /* for restore */
             buffer.limit(buffer.position() + valueLen);
 
             /* @see User_var_log_event::print */
-            switch (type)
-            {
+            switch (type) {
                 case REAL_RESULT:
                     value = Double.valueOf(buffer.getDouble64()); // float8get
                     break;
                 case INT_RESULT:
-                if (valueLen == 8)
-                    value = Long.valueOf(buffer.getLong64()); // !uint8korr
-                else if (valueLen == 4)
-                    value = Long.valueOf(buffer.getUint32());
-                else
-                    throw new IOException("Error INT_RESULT length: "
-                            + valueLen);
+                    if (valueLen == 8) value = Long.valueOf(buffer.getLong64()); // !uint8korr
+                    else if (valueLen == 4) value = Long.valueOf(buffer.getUint32());
+                    else throw new IOException("Error INT_RESULT length: " + valueLen);
                     break;
                 case DECIMAL_RESULT:
                     final int precision = buffer.getInt8();
@@ -127,19 +114,13 @@ public final class UserVarLogEvent extends LogEvent
         }
     }
 
-    public final String getQuery()
-    {
-        if (value == null)
-        {
+    public final String getQuery() {
+        if (value == null) {
             return "SET @" + name + " := NULL";
-        }
-        else if (type == STRING_RESULT)
-        {
+        } else if (type == STRING_RESULT) {
             // TODO: do escaping !?
             return "SET @" + name + " := \'" + value + '\'';
-        }
-        else
-        {
+        } else {
             return "SET @" + name + " := " + String.valueOf(value);
         }
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class WriteRowsLogEvent extends RowsLogEvent
-{
-    public WriteRowsLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+public final class WriteRowsLogEvent extends RowsLogEvent {
+
+    public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java

@@ -10,23 +10,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class XidLogEvent extends LogEvent
-{
+public final class XidLogEvent extends LogEvent {
+
     private final long xid;
 
-    public XidLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
+    public XidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
 
         /* The Post-Header is empty. The Variable Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
-                + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
         xid = buffer.getLong64(); // !uint8korr
     }
 
-    public final long getXid()
-    {
+    public final long getXid() {
         return xid;
     }
 }

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java

@@ -10,8 +10,8 @@ import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
 import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 
 public class BaseLogFetcherTest {
@@ -20,36 +20,38 @@ public class BaseLogFetcherTest {
     protected Charset charset        = Charset.forName("utf-8");
 
     protected void parseQueryEvent(QueryLogEvent event) {
-        System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
+        System.out.println(String.format("================> binlog[%s:%s] , name[%s]",
+            binlogFileName,
             event.getHeader().getLogPos() - event.getHeader().getEventLen(),
             event.getCatalog()));
         System.out.println("sql : " + event.getQuery());
     }
 
     protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
-        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
         System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
     }
 
     protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
-        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
         System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
     }
 
     protected void parseXidEvent(XidLogEvent event) throws Exception {
-        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
         System.out.println("xid : " + event.getXid());
     }
 
-
     protected void parseRowsEvent(RowsLogEvent event) {
         try {
-            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
+            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
+                binlogFileName,
                 event.getHeader().getLogPos() - event.getHeader().getEventLen(),
-                                             event.getTable().getDbName(), event.getTable().getTableName()));
+                event.getTable().getDbName(),
+                event.getTable().getTableName()));
             RowsLogBuffer buffer = event.getRowsBuf(charset.name());
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
@@ -90,6 +92,10 @@ public class BaseLogFetcherTest {
         final ColumnInfo[] columnInfo = map.getColumnInfo();
 
         for (int i = 0; i < columnCnt; i++) {
+            if (!cols.get(i)) {
+                continue;
+            }
+
             ColumnInfo info = columnInfo[i];
             buffer.nextValue(info.type, info.meta);
 

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java

@@ -5,8 +5,7 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.Statement;
 
-import junit.framework.Assert;
-
+import org.junit.Assert;
 import org.junit.Test;
 
 import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
@@ -25,12 +24,12 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
         DirectLogFetcher fecther = new DirectLogFetcher();
         try {
             Class.forName("com.mysql.jdbc.Driver");
-            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "xxxxx", "xxxxx");
+            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello");
             Statement statement = connection.createStatement();
             statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
             statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
 
-            fecther.open(connection, "mysql-bin.000003", 4L, 2);
+            fecther.open(connection, "mysql-bin.000001", 4L, 2);
 
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
@@ -39,7 +38,8 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
                 event = decoder.decode(fecther, context);
 
                 if (event == null) {
-                    throw new RuntimeException("parse failed");
+                    continue;
+                    // throw new RuntimeException("parse failed");
                 }
 
                 int eventType = event.getHeader().getType();

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java

@@ -4,8 +4,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URL;
 
-import junit.framework.Assert;
-
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -27,7 +26,6 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
         URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
         File dummyFile = new File(url.getFile());
         directory = new File(dummyFile.getParent() + "/binlog").getPath();
-        // directory = "/home/jianghang/tmp/binlog";
     }
 
     @Test
@@ -37,8 +35,8 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
 
-            File current = new File(directory, "mysql-bin.000006");
-            fetcher.open(current);
+            File current = new File(directory, "mysql-bin.000001");
+            fetcher.open(current, 2051L);
             context.setLogPosition(new LogPosition(current.getName()));
 
             while (fetcher.fetch()) {
@@ -48,8 +46,7 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
                     int eventType = event.getHeader().getType();
                     switch (eventType) {
                         case LogEvent.ROTATE_EVENT:
-                             binlogFileName = ((RotateLogEvent)
-                             event).getFilename();
+                            binlogFileName = ((RotateLogEvent) event).getFilename();
                             break;
                         case LogEvent.WRITE_ROWS_EVENT_V1:
                         case LogEvent.WRITE_ROWS_EVENT:

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java

@@ -5,14 +5,11 @@ import java.math.BigInteger;
 
 import junit.framework.TestCase;
 
-import com.taobao.tddl.dbsync.binlog.LogBuffer;
+public class LogBufferTest extends TestCase {
 
-public class LogBufferTest extends TestCase
-{
     public static final int LOOP = 10000;
 
-    public void testSigned()
-    {
+    public void testSigned() {
         byte[] array = { 0, 0, 0, (byte) 0xff };
 
         LogBuffer buffer = new LogBuffer(array, 0, array.length);
@@ -24,17 +21,15 @@ public class LogBufferTest extends TestCase
         System.out.println(buffer.getUint24(1));
     }
 
-    public void testBigInteger()
-    {
-        byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
-                (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
+    public void testBigInteger() {
+        byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
+                (byte) 0xff };
 
         LogBuffer buffer = new LogBuffer(array, 0, array.length);
 
         long tt1 = 0;
         long l1 = 0;
-        for (int i = 0; i < LOOP; i++)
-        {
+        for (int i = 0; i < LOOP; i++) {
             final long t1 = System.nanoTime();
             l1 = buffer.getLong64(0);
             tt1 += System.nanoTime() - t1;
@@ -45,8 +40,7 @@ public class LogBufferTest extends TestCase
 
         long tt2 = 0;
         BigInteger l2 = null;
-        for (int i = 0; i < LOOP; i++)
-        {
+        for (int i = 0; i < LOOP; i++) {
             final long t2 = System.nanoTime();
             l2 = buffer.getUlong64(0);
             tt2 += System.nanoTime() - t2;
@@ -57,18 +51,15 @@ public class LogBufferTest extends TestCase
     }
 
     /* Reads big-endian integer from no more than 4 bytes */
-    private static int convertNBytesToInt(byte[] buffer, int offset, int length)
-    {
+    private static int convertNBytesToInt(byte[] buffer, int offset, int length) {
         int ret = 0;
-        for (int i = offset; i < (offset + length); i++)
-        {
+        for (int i = offset; i < (offset + length); i++) {
             ret = (ret << 8) | (0xff & buffer[i]);
         }
         return ret;
     }
 
-    private static int convert4BytesToInt(byte[] buffer, int offset)
-    {
+    private static int convert4BytesToInt(byte[] buffer, int offset) {
         int value;
         value = (0xff & buffer[offset + 3]);
         value += (0xff & buffer[offset + 2]) << 8;
@@ -77,24 +68,20 @@ public class LogBufferTest extends TestCase
         return value;
     }
 
-    public static short convert1ByteToShort(byte[] buffer, int offset)
-    {
+    public static short convert1ByteToShort(byte[] buffer, int offset) {
         short value;
         value = (short) buffer[offset + 0];
         return value;
     }
 
-    public static short convert2bytesToShort(byte[] buffer, int offset)
-    {
+    public static short convert2bytesToShort(byte[] buffer, int offset) {
         short value;
         value = (short) (buffer[offset + 0] << 8);
         value += (short) (buffer[offset + 1] & 0xff);
         return value;
     }
 
-    public static final BigDecimal extractDecimal(byte[] buffer, int precision,
-            int scale)
-    {
+    public static final BigDecimal extractDecimal(byte[] buffer, int precision, int scale) {
         //
         // Decimal representation in binlog seems to be as follows:
         // 1 byte - 'precision'
@@ -129,34 +116,28 @@ public class LogBufferTest extends TestCase
         int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
 
         // how many bytes are used to represent given amount of digits?
-        int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
-                + LogBuffer.dig2bytes[intg0x];
-        int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
-                + LogBuffer.dig2bytes[frac0x];
+        int integerSize = intg0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[intg0x];
+        int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[frac0x];
 
         int bin_size = integerSize + decimalSize; // total bytes
         byte[] d_copy = new byte[bin_size];
 
-        if (bin_size > buffer.length)
-        {
-            throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
-                    + bin_size + ", available bytes: " + buffer.length);
+        if (bin_size > buffer.length) {
+            throw new ArrayIndexOutOfBoundsException("Calculated bin_size: " + bin_size + ", available bytes: "
+                                                     + buffer.length);
         }
 
         // Invert first bit
         d_copy[0] = buffer[0];
         d_copy[0] ^= 0x80;
-        if (sign == -1)
-        {
+        if (sign == -1) {
             // Invert every byte
             d_copy[0] ^= 0xFF;
         }
 
-        for (int i = 1; i < bin_size; i++)
-        {
+        for (int i = 1; i < bin_size; i++) {
             d_copy[i] = buffer[i];
-            if (sign == -1)
-            {
+            if (sign == -1) {
                 // Invert every byte
                 d_copy[i] ^= 0xFF;
             }
@@ -167,62 +148,49 @@ public class LogBufferTest extends TestCase
 
         BigDecimal intPart = new BigDecimal(0);
 
-        if (offset > 0)
-            intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
+        if (offset > 0) intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
 
-        while (offset < integerSize)
-        {
-            intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
-                    BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
+        while (offset < integerSize) {
+            intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(BigDecimal.valueOf(convert4BytesToInt(d_copy,
+                offset)));
             offset += 4;
         }
 
         // Decimal part
         BigDecimal fracPart = new BigDecimal(0);
         int shift = 0;
-        for (int i = 0; i < frac0; i++)
-        {
+        for (int i = 0; i < frac0; i++) {
             shift += LogBuffer.DIG_PER_DEC1;
-            fracPart = fracPart.add(BigDecimal.valueOf(
-                    convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
+            fracPart = fracPart.add(BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
             offset += 4;
         }
 
-        if (LogBuffer.dig2bytes[frac0x] > 0)
-        {
-            fracPart = fracPart.add(BigDecimal.valueOf(
-                    convertNBytesToInt(d_copy, offset,
-                            LogBuffer.dig2bytes[frac0x])).movePointLeft(
-                    shift + frac0x));
+        if (LogBuffer.dig2bytes[frac0x] > 0) {
+            fracPart = fracPart.add(BigDecimal.valueOf(convertNBytesToInt(d_copy, offset, LogBuffer.dig2bytes[frac0x]))
+                .movePointLeft(shift + frac0x));
         }
 
         return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
     }
 
-    public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
-            0x38, (byte) 0xb0, 0x60, 0x00 };
+    public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b, 0x38, (byte) 0xb0, 0x60, 0x00 };
 
-    public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
-            (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
-            (byte) 0xa0, (byte) 0xff };
+    public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7,
+            (byte) 0x4f, (byte) 0xa0, (byte) 0xff };
 
     public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
 
     public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
 
-    public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
-            -108                     };
+    public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96, -108 };
 
-    public void testBigDecimal() throws InterruptedException
-    {
-        do
-        {
+    public void testBigDecimal() throws InterruptedException {
+        do {
             System.out.println("old extract decimal: ");
 
             long tt1 = 0;
             BigDecimal bd1 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t1 = System.nanoTime();
                 bd1 = extractDecimal(array2, 19, 10);
                 tt1 += System.nanoTime() - t1;
@@ -233,8 +201,7 @@ public class LogBufferTest extends TestCase
 
             long tt2 = 0;
             BigDecimal bd2 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t2 = System.nanoTime();
                 bd2 = extractDecimal(array1, 19, 10);
                 tt2 += System.nanoTime() - t2;
@@ -245,8 +212,7 @@ public class LogBufferTest extends TestCase
 
             long tt3 = 0;
             BigDecimal bd3 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t3 = System.nanoTime();
                 bd3 = extractDecimal(array3, 18, 6);
                 tt3 += System.nanoTime() - t3;
@@ -257,8 +223,7 @@ public class LogBufferTest extends TestCase
 
             long tt4 = 0;
             BigDecimal bd4 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t4 = System.nanoTime();
                 bd4 = extractDecimal(array4, 18, 6);
                 tt4 += System.nanoTime() - t4;
@@ -269,8 +234,7 @@ public class LogBufferTest extends TestCase
 
             long tt5 = 0;
             BigDecimal bd5 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t5 = System.nanoTime();
                 bd5 = extractDecimal(array5, 18, 6);
                 tt5 += System.nanoTime() - t5;
@@ -278,11 +242,9 @@ public class LogBufferTest extends TestCase
             System.out.print(tt5 / LOOP);
             System.out.print("ns >> ");
             System.out.println(bd5);
-        }
-        while (false);
+        } while (false);
 
-        do
-        {
+        do {
             System.out.println("new extract decimal: ");
 
             LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
@@ -293,8 +255,7 @@ public class LogBufferTest extends TestCase
 
             long tt1 = 0;
             BigDecimal bd1 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t1 = System.nanoTime();
                 bd1 = buffer1.getDecimal(0, 19, 10);
                 tt1 += System.nanoTime() - t1;
@@ -305,8 +266,7 @@ public class LogBufferTest extends TestCase
 
             long tt2 = 0;
             BigDecimal bd2 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t2 = System.nanoTime();
                 bd2 = buffer2.getDecimal(0, 19, 10);
                 tt2 += System.nanoTime() - t2;
@@ -317,8 +277,7 @@ public class LogBufferTest extends TestCase
 
             long tt3 = 0;
             BigDecimal bd3 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t3 = System.nanoTime();
                 bd3 = buffer3.getDecimal(0, 18, 6);
                 tt3 += System.nanoTime() - t3;
@@ -329,8 +288,7 @@ public class LogBufferTest extends TestCase
 
             long tt4 = 0;
             BigDecimal bd4 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t4 = System.nanoTime();
                 bd4 = buffer4.getDecimal(0, 18, 6);
                 tt4 += System.nanoTime() - t4;
@@ -341,8 +299,7 @@ public class LogBufferTest extends TestCase
 
             long tt5 = 0;
             BigDecimal bd5 = null;
-            for (int i = 0; i < LOOP; i++)
-            {
+            for (int i = 0; i < LOOP; i++) {
                 final long t5 = System.nanoTime();
                 bd5 = buffer5.getDecimal(0, 18, 6);
                 tt5 += System.nanoTime() - t5;
@@ -350,7 +307,6 @@ public class LogBufferTest extends TestCase
             System.out.print(tt5 / LOOP);
             System.out.print("ns >> ");
             System.out.println(bd5);
-        }
-        while (false);
+        } while (false);
     }
 }

deployer/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

deployer/src/main/bin/stop.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 
 cygwin=false;
+linux=false;
 case "`uname`" in
     CYGWIN*)
         cygwin=true
         ;;
+    Linux*)
+    	linux=true
+    	;;
 esac
 
 get_pid() {	
@@ -15,11 +19,19 @@ get_pid() {
         JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
         JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
     else
+    	if $linux; then
 	        if [ ! -z "$PID" ]; then
 	        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
 		    else 
 		        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
 	        fi
+	    else
+	    	if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    fi
     fi
     echo $JAVA_PID;
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java

@@ -5,6 +5,7 @@ import java.util.Properties;
 
 import org.I0Itec.zkclient.IZkStateListener;
 import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zookeeper.Watcher.Event.KeeperState;
@@ -15,6 +16,7 @@ import org.springframework.beans.factory.BeanFactory;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
+import com.alibaba.otter.canal.common.CanalException;
 import com.alibaba.otter.canal.common.utils.AddressUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
@@ -32,11 +34,12 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
 import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
 import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
 import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
-import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
 import com.alibaba.otter.canal.server.exception.CanalServerException;
 import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * canal调度控制器
@@ -58,7 +61,7 @@ public class CanalController {
     private boolean                                  autoScan = true;
     private InstanceAction                           defaultAction;
     private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
-    private CanalServerWithEmbeded                   embededCanalServer;
+    private CanalServerWithEmbedded                  embededCanalServer;
     private CanalServerWithNetty                     canalServer;
 
     private CanalInstanceGenerator                   instanceGenerator;
@@ -69,7 +72,7 @@ public class CanalController {
     }
 
     public CanalController(final Properties properties){
-        managerClients = new MapMaker().makeComputingMap(new Function<String, CanalConfigClient>() {
+        managerClients = MigrateMap.makeComputingMap(new Function<String, CanalConfigClient>() {
 
             public CanalConfigClient apply(String managerAddress) {
                 return getManagerClient(managerAddress);
@@ -86,9 +89,9 @@ public class CanalController {
         cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
         ip = getProperty(properties, CanalConstants.CANAL_IP);
         port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT));
-        embededCanalServer = new CanalServerWithEmbeded();
+        embededCanalServer = CanalServerWithEmbedded.instance();
         embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
-        canalServer = new CanalServerWithNetty(embededCanalServer);
+        canalServer = CanalServerWithNetty.instance();
         canalServer.setIp(ip);
         canalServer.setPort(port);
 
@@ -106,7 +109,7 @@ public class CanalController {
 
         final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
         ServerRunningMonitors.setServerData(serverData);
-        ServerRunningMonitors.setRunningMonitors(new MapMaker().makeComputingMap(new Function<String, ServerRunningMonitor>() {
+        ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
 
             public ServerRunningMonitor apply(final String destination) {
                 ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
@@ -146,6 +149,11 @@ public class CanalController {
                                     public void handleNewSession() throws Exception {
                                         initCid(path);
                                     }
+
+                                    @Override
+                                    public void handleSessionEstablishmentError(Throwable error) throws Exception {
+                                        logger.error("failed to connect to zookeeper", error);
+                                    }
                                 });
                             }
                         } finally {
@@ -170,6 +178,8 @@ public class CanalController {
                 if (zkclientx != null) {
                     runningMonitor.setZkClient(zkclientx);
                 }
+                // 触发创建一下cid节点
+                runningMonitor.init();
                 return runningMonitor;
             }
         }));
@@ -182,14 +192,15 @@ public class CanalController {
                 public void start(String destination) {
                     InstanceConfig config = instanceConfigs.get(destination);
                     if (config == null) {
-                        config = new InstanceConfig(globalInstanceConfig);
+                        // 重新读取一下instance config
+                        config = parseInstanceConfig(properties, destination);
                         instanceConfigs.put(destination, config);
                     }
 
-                    if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
+                    if (!embededCanalServer.isStart(destination)) {
                         // HA机制启动
                         ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
-                        if (!runningMonitor.isStart()) {
+                        if (!config.getLazy() && !runningMonitor.isStart()) {
                             runningMonitor.start();
                         }
                     }
@@ -214,7 +225,7 @@ public class CanalController {
                 }
             };
 
-            instanceConfigMonitors = new MapMaker().makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
+            instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
 
                 public InstanceConfigMonitor apply(InstanceMode mode) {
                     int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
@@ -228,7 +239,13 @@ public class CanalController {
                         if (StringUtils.isEmpty(rootDir)) {
                             rootDir = "../conf";
                         }
+
+                        if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
                             monitor.setRootConf(rootDir);
+                        } else {
+                            // eclipse debug模式
+                            monitor.setRootConf("src/main/resources/");
+                        }
                         return monitor;
                     } else if (mode.isManager()) {
                         return new ManagerInstanceConfigMonitor();
@@ -283,6 +300,9 @@ public class CanalController {
                             System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination);
                             instanceGenerator.setBeanFactory(getBeanFactory(config.getSpringXml()));
                             return instanceGenerator.generate(destination);
+                        } catch (Throwable e) {
+                            logger.error("generator instance failed.", e);
+                            throw new CanalException(e);
                         } finally {
                             System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, "");
                         }
@@ -368,6 +388,11 @@ public class CanalController {
                 public void handleNewSession() throws Exception {
                     initCid(path);
                 }
+
+                @Override
+                public void handleSessionEstablishmentError(Throwable error) throws Exception {
+                    logger.error("failed to connect to zookeeper", error);
+                }
             });
         }
         // 优先启动embeded服务
@@ -377,16 +402,16 @@ public class CanalController {
             final String destination = entry.getKey();
             InstanceConfig config = entry.getValue();
             // 创建destination的工作节点
-            if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
+            if (!embededCanalServer.isStart(destination)) {
                 // HA机制启动
                 ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
-                if (!runningMonitor.isStart()) {
+                if (!config.getLazy() && !runningMonitor.isStart()) {
                     runningMonitor.start();
                 }
             }
 
             if (autoScan) {
-                instanceConfigMonitors.get(config.getMode()).regeister(destination, defaultAction);
+                instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction);
             }
         }
 
@@ -436,6 +461,9 @@ public class CanalController {
                 String parentDir = path.substring(0, path.lastIndexOf('/'));
                 zkclientx.createPersistent(parentDir, true);
                 zkclientx.createEphemeral(path);
+            } catch (ZkNodeExistsException e) {
+                // ignore
+                // 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s
             }
 
         }

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java

@@ -4,7 +4,6 @@ import java.io.FileInputStream;
 import java.util.Properties;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,6 +20,10 @@ public class CanalLauncher {
 
     public static void main(String[] args) throws Throwable {
         try {
+            logger.info("## set default uncaught exception handler");
+            setGlobalUncaughtExceptionHandler();
+
+            logger.info("## load canal configurations");
             String conf = System.getProperty("canal.conf", "classpath:canal.properties");
             Properties properties = new Properties();
             if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
@@ -41,8 +44,7 @@ public class CanalLauncher {
                         logger.info("## stop the canal server");
                         controller.stop();
                     } catch (Throwable e) {
-                        logger.warn("##something goes wrong when stopping canal Server:\n{}",
-                                    ExceptionUtils.getFullStackTrace(e));
+                        logger.warn("##something goes wrong when stopping canal Server:", e);
                     } finally {
                         logger.info("## canal server is down.");
                     }
@@ -50,9 +52,17 @@ public class CanalLauncher {
 
             });
         } catch (Throwable e) {
-            logger.error("## Something goes wrong when starting up the canal Server:\n{}",
-                         ExceptionUtils.getFullStackTrace(e));
+            logger.error("## Something goes wrong when starting up the canal Server:", e);
             System.exit(0);
         }
     }
+
+    private static void setGlobalUncaughtExceptionHandler() {
+        Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+            @Override
+            public void uncaughtException(Thread t, Throwable e) {
+                logger.error("UnCaughtException", e);
+            }
+        });
+    }
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java

@@ -11,15 +11,15 @@ public interface InstanceAction {
     /**
      * 启动destination
      */
-    public void start(String destination);
+    void start(String destination);
 
     /**
      * 停止destination
      */
-    public void stop(String destination);
+    void stop(String destination);
 
     /**
      * 重载destination，可能需要stop,start操作，或者只是更新下内存配置
      */
-    public void reload(String destination);
+    void reload(String destination);
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java

@@ -10,7 +10,7 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
  */
 public interface InstanceConfigMonitor extends CanalLifeCycle {
 
-    public void regeister(String destination, InstanceAction action);
+    void register(String destination, InstanceAction action);
 
-    public void unRegeister(String destination);
+    void unregister(String destination);
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java

@@ -9,11 +9,11 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
  */
 public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {
 
-    public void regeister(String destination, InstanceAction action) {
+    public void register(String destination, InstanceAction action) {
 
     }
 
-    public void unRegeister(String destination) {
+    public void unregister(String destination) {
 
     }
 

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java

@@ -13,7 +13,6 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -24,6 +23,7 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
 import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * 监听基于spring配置的instance变化
@@ -39,7 +39,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
     private long                             scanIntervalInSecond = 5;
     private InstanceAction                   defaultAction        = null;
     private Map<String, InstanceAction>      actions              = new MapMaker().makeMap();
-    private Map<String, InstanceConfigFiles> lastFiles            = new MapMaker().makeComputingMap(new Function<String, InstanceConfigFiles>() {
+    private Map<String, InstanceConfigFiles> lastFiles            = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>() {
 
                                                                       public InstanceConfigFiles apply(String destination) {
                                                                           return new InstanceConfigFiles(destination);
@@ -48,6 +48,8 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
     private ScheduledExecutorService         executor             = Executors.newScheduledThreadPool(1,
                                                                       new NamedThreadFactory("canal-instance-scan"));
 
+    private volatile boolean                 isFirst              = true;
+
     public void start() {
         super.start();
         Assert.notNull(rootConf, "root conf dir is null!");
@@ -57,6 +59,9 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
             public void run() {
                 try {
                     scan();
+                    if (isFirst) {
+                        isFirst = false;
+                    }
                 } catch (Throwable e) {
                     logger.error("scan failed", e);
                 }
@@ -72,7 +77,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         lastFiles.clear();
     }
 
-    public void regeister(String destination, InstanceAction action) {
+    public void register(String destination, InstanceAction action) {
         if (action != null) {
             actions.put(destination, action);
         } else {
@@ -80,7 +85,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         }
     }
 
-    public void unRegeister(String destination) {
+    public void unregister(String destination) {
         actions.remove(destination);
     }
 
@@ -121,13 +126,18 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
 
             if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
                 // 存在合法的instance.properties，并且第一次添加时，进行启动操作
-                notifyStart(instanceDir, destination);
+                notifyStart(instanceDir, destination, instanceConfigs);
             } else if (actions.containsKey(destination)) {
                 // 历史已经启动过
                 if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties
                     notifyStop(destination);
                 } else {
                     InstanceConfigFiles lastFile = lastFiles.get(destination);
+                    // 历史启动过 所以配置文件信息必然存在
+                    if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
+                        logger.error("[{}] is started, but not found instance file info.", destination);
+                    }
+
                     boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles());
                     // 通知变化
                     if (hasChanged) {
@@ -160,13 +170,22 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         }
     }
 
-    private void notifyStart(File instanceDir, String destination) {
+    private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) {
         try {
             defaultAction.start(destination);
             actions.put(destination, defaultAction);
+
+            // 启动成功后记录配置文件信息
+            InstanceConfigFiles lastFile = lastFiles.get(destination);
+            List<FileInfo> newFileInfo = new ArrayList<FileInfo>();
+            for (File instanceConfig : instanceConfigs) {
+                newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
+            }
+            lastFile.setInstanceFiles(newFileInfo);
+
             logger.info("auto notify start {} successful.", destination);
         } catch (Throwable e) {
-            logger.error("scan add found[{}] but start failed", destination, ExceptionUtils.getFullStackTrace(e));
+            logger.error(String.format("scan add found[%s] but start failed", destination), e);
         }
     }
 
@@ -174,9 +193,10 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         InstanceAction action = actions.remove(destination);
         try {
             action.stop(destination);
+            lastFiles.remove(destination);
             logger.info("auto notify stop {} successful.", destination);
         } catch (Throwable e) {
-            logger.error("scan delete found[{}] but stop failed", destination, ExceptionUtils.getFullStackTrace(e));
+            logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
             actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除
         }
     }
@@ -188,9 +208,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
                 action.reload(destination);
                 logger.info("auto notify reload {} successful.", destination);
             } catch (Throwable e) {
-                logger.error("scan reload found[{}] but reload failed",
-                    destination,
-                    ExceptionUtils.getFullStackTrace(e));
+                logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
             }
         }
     }
@@ -293,5 +311,4 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         }
 
     }
-
 }

deployer/src/main/resources/canal.properties

@@ -36,10 +36,16 @@ canal.instance.network.sendBufferSize = 16384
 canal.instance.network.soTimeout = 30
 
 # binlog filter config
+canal.instance.filter.druid.ddl = true
 canal.instance.filter.query.dcl = false
 canal.instance.filter.query.dml = false
 canal.instance.filter.query.ddl = false
 canal.instance.filter.table.error = false
+canal.instance.filter.rows = false
+
+# binlog format/image check
+canal.instance.binlog.format = ROW,STATEMENT,MIXED 
+canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
 
 # binlog ddl isolation
 canal.instance.get.ddl.isolation = false
@@ -54,9 +60,13 @@ canal.conf.dir = ../conf
 canal.auto.scan = true
 canal.auto.scan.interval = 5
 
+canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml
+#canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml
+
 canal.instance.global.mode = spring 
 canal.instance.global.lazy = false
 #canal.instance.global.manager.address = 127.0.0.1:1099
+#canal.instance.global.spring.xml = classpath:spring/local-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
 canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml

deployer/src/main/resources/example/instance.properties

@@ -1,27 +1,33 @@
 #################################################
 ## mysql serverId
-canal.instance.mysql.slaveId = 1234
-
+canal.instance.mysql.slaveId=0
 # position info
 canal.instance.master.address=127.0.0.1:3306
 canal.instance.master.journal.name=
 canal.instance.master.position=
 canal.instance.master.timestamp=
 
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+canal.instance.tsdb.dbUsername=canal
+canal.instance.tsdb.dbPassword=canal
+
+
 #canal.instance.standby.address =
 #canal.instance.standby.journal.name =
 #canal.instance.standby.position = 
 #canal.instance.standby.timestamp = 
-
 # username/password
 canal.instance.dbUsername=canal
 canal.instance.dbPassword=canal
-canal.instance.defaultDatabaseName =
+canal.instance.defaultDatabaseName=test
 canal.instance.connectionCharset=UTF-8
-
 # table regex
 canal.instance.filter.regex=.*\\..*
 # table black regex
 canal.instance.filter.black.regex=
-
 #################################################

deployer/src/main/resources/example/rds_instance.properties

@@ -0,0 +1,32 @@
+#################################################
+# rds openapi binlog
+canal.instance.rds.open.url=https://rds.aliyuncs.com/
+canal.instance.rds.open.accesskey=
+canal.instance.rds.open.secretkey=
+canal.instance.rds.instanceId=
+canal.instance.rds.startTime=
+canal.instance.rds.endTime=
+
+# local binlog dir
+canal.instance.parser.directory=${canal.file.data.dir:../conf}/${canal.instance.destination:}/binlog
+# position info
+canal.instance.master.address=127.0.0.1:3306
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+#canal.instance.tsdb.dbUsername=canal
+#canal.instance.tsdb.dbPassword=canal
+
+# username/password
+canal.instance.dbUsername=canal
+canal.instance.dbPassword=canal
+canal.instance.defaultDatabaseName=test
+canal.instance.connectionCharset=UTF-8
+# table regex
+canal.instance.filter.regex=test\\..*
+# table black regex
+canal.instance.filter.black.regex=
+#################################################

deployer/src/main/resources/logback.xml

@@ -75,9 +75,7 @@
     </logger>
     
 	<root level="WARN">
-		<!--  
 		<appender-ref ref="STDOUT"/>
-		-->
 		<appender-ref ref="CANAL-ROOT" />
 	</root>
 </configuration>

deployer/src/main/resources/spring/default-instance.xml

@@ -85,7 +85,7 @@
 
 	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -128,14 +128,14 @@
 		<!-- 解析位点记录 -->
 		<property name="logPositionManager">
 			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
-				<property name="primary">
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
-				</property>
-				<property name="failback">
+				</constructor-arg>
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
-						<property name="metaManager" ref="metaManager" />
+						<constructor-arg ref="metaManager"/>
 					</bean>
-				</property>
+				</constructor-arg>
 			</bean>
 		</property>
 		
@@ -178,6 +178,14 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
 	</bean>
 </beans>

deployer/src/main/resources/spring/file-instance.xml

@@ -71,7 +71,7 @@
 
 	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -113,14 +113,14 @@
 		<!-- 解析位点记录 -->
 		<property name="logPositionManager">
 			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
-				<property name="primary">
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
-				</property>
-				<property name="failback">
+				</constructor-arg>
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
-						<property name="metaManager" ref="metaManager" />
+						<constructor-arg ref="metaManager"/>
 					</bean>
-				</property>
+				</constructor-arg>
 			</bean>
 		</property>
 		
@@ -163,6 +163,14 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
+
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
 	</bean>
 </beans>

deployer/src/main/resources/spring/group-instance.xml

@@ -77,7 +77,7 @@
 
 	<bean id="eventParser1" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -160,12 +160,15 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
 	</bean>
 	
 	<bean id="eventParser2" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -248,6 +251,10 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
 	</bean>
 </beans>

deployer/src/main/resources/spring/local-instance.xml

@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
+		<property name="destination" value="${canal.instance.destination}" />
+		<property name="eventParser">
+			<ref local="eventParser" />
+		</property>
+		<property name="eventSink">
+			<ref local="eventSink" />
+		</property>
+		<property name="eventStore">
+			<ref local="eventStore" />
+		</property>
+		<property name="metaManager">
+			<ref local="metaManager" />
+		</property>
+		<property name="alarmHandler">
+			<ref local="alarmHandler" />
+		</property>
+	</bean>
+	
+	<!-- 报警处理类 -->
+	<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
+	
+	<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
+		<property name="dataDir" value="${canal.file.data.dir:../conf}" />
+		<property name="period" value="${canal.file.flush.period:1000}" />
+	</bean>
+	
+	<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
+		<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
+		<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
+		<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
+		<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
+	</bean>
+	
+	<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
+		<property name="eventStore" ref="eventStore" />
+	</bean>
+
+	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser">
+		<property name="destination" value="${canal.instance.destination}" />			
+		<property name="alarmHandler" ref="alarmHandler" />
+
+		<!-- 解析过滤处理 -->
+		<property name="eventFilter">
+			<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
+				<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
+			</bean>
+		</property>
+		
+		<property name="eventBlackFilter">
+			<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
+				<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
+				<constructor-arg index="1" value="false" />
+			</bean>
+		</property>
+		
+		<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
+		<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
+			
+		<!-- 解析编码 -->
+		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
+	
+		<!-- 解析位点记录 -->
+		<property name="logPositionManager">
+			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
+				<constructor-arg>
+					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
+				</constructor-arg>
+				<constructor-arg>
+					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
+						<constructor-arg ref="metaManager"/>
+					</bean>
+				</constructor-arg>
+			</bean>
+		</property>
+		
+		<!-- 解析数据库信息 -->
+		<property name="masterInfo">
+			<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
+				<property name="address" value="${canal.instance.master.address}" />
+				<property name="username" value="${canal.instance.dbUsername:retl}" />
+				<property name="password" value="${canal.instance.dbPassword:retl}" />
+				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
+			</bean>
+		</property>
+		
+		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
+		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
+		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
+		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="needWait" value="${canal.instance.parser.needWait:false}"/>
+		<property name="directory" value="${canal.instance.parser.directory:}"/>
+		
+		<!-- rds相关 -->
+		<property name="url" value="${canal.instance.rds.open.url:}"/>
+		<property name="accesskey" value="${canal.instance.rds.open.accesskey:}"/>
+		<property name="secretkey" value="${canal.instance.rds.open.secretkey:}"/>
+		<property name="instanceId" value="${canal.instance.rds.instanceId:}"/>
+		<property name="startTime" value="${canal.instance.rds.startTime:}"/>
+		<property name="endTime" value="${canal.instance.rds.endTime:}"/>
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
+	</bean>
+</beans>

deployer/src/main/resources/spring/memory-instance.xml

@@ -68,7 +68,7 @@
 
 	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -151,6 +151,14 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
 	</bean>
 </beans>

deployer/src/main/resources/spring/tsdb/h2-tsdb.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+	
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<!-- 基于db的实现 -->
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
+		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
+	</bean>
+	
+    <bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
+        <property name="driverClassName" value="org.h2.Driver" />
+		<property name="url" value="${canal.instance.tsdb.url:}" />
+		<property name="username" value="${canal.instance.tsdb.dbUsername:canal}" />
+		<property name="password" value="${canal.instance.tsdb.dbPassword:canal}" />
+      	<property name="maxActive" value="30" />
+        <property name="initialSize" value="0" />
+        <property name="minIdle" value="1" />
+        <property name="maxWait" value="10000" />
+        <property name="timeBetweenEvictionRunsMillis" value="60000" />
+        <property name="minEvictableIdleTimeMillis" value="300000" />
+        <property name="testWhileIdle" value="true" />
+        <property name="testOnBorrow" value="false" />
+        <property name="testOnReturn" value="false" />
+        <property name="useUnfairLock" value="true" />
+	</bean>
+
+    <bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
+        <property name="dataSource" ref="dataSource"/>
+        <property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
+    </bean>
+
+    <bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+
+    <bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+</beans>

deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+	
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<!-- 基于db的实现 -->
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
+		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
+	</bean>
+	
+    <bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
+        <property name="driverClassName" value="com.mysql.jdbc.Driver" />
+		<property name="url" value="${canal.instance.tsdb.url:}" />
+		<property name="username" value="${canal.instance.tsdb.dbUsername:canal}" />
+		<property name="password" value="${canal.instance.tsdb.dbPassword:canal}" />
+        <property name="maxActive" value="30" />
+        <property name="initialSize" value="0" />
+        <property name="minIdle" value="1" />
+        <property name="maxWait" value="10000" />
+        <property name="timeBetweenEvictionRunsMillis" value="60000" />
+        <property name="minEvictableIdleTimeMillis" value="300000" />
+        <property name="validationQuery" value="SELECT 1" />
+        <property name="exceptionSorterClassName" value="com.alibaba.druid.pool.vendor.MySqlExceptionSorter" />
+        <property name="validConnectionCheckerClassName" value="com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker" />
+        <property name="testWhileIdle" value="true" />
+        <property name="testOnBorrow" value="false" />
+        <property name="testOnReturn" value="false" />
+        <property name="useUnfairLock" value="true" />
+	</bean>
+
+    <bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
+        <property name="dataSource" ref="dataSource"/>
+        <property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
+    </bean>
+
+    <bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+
+    <bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+</beans>

deployer/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE sqlMapConfig PUBLIC "-//iBATIS.com//DTD SQL Map Config 2.0//EN"
+        "http://www.ibatis.com/dtd/sql-map-config-2.dtd">
+<sqlMapConfig>
+    <settings useStatementNamespaces="true"/>
+    <sqlMap resource="spring/tsdb/sql-map/sqlmap_history.xml"/>
+    <sqlMap resource="spring/tsdb/sql-map/sqlmap_snapshot.xml"/>
+</sqlMapConfig>

deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
+<sqlMap namespace="meta_history">
+    <typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO"/>
+    <sql id="allColumns">
+        <![CDATA[
+		gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,sql_schema,sql_table,sql_text,sql_type,extra
+        ]]>
+    </sql>
+    <sql id="allVOColumns">
+        <![CDATA[
+		a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
+		a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
+		a.use_schema as useSchema,a.sql_schema as sqlSchema,a.sql_table as sqlTable,a.sql_text as sqlText,a.sql_type as sqlType,a.extra as extra
+        ]]>
+    </sql>
+
+    <select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaHistoryDO">
+        select
+        <include refid="allVOColumns"/>
+        from meta_history a
+        <![CDATA[
+        where destination = #destination# and binlog_timestamp >= #snapshotTimestamp# and binlog_timestamp <= #timestamp#
+        order by binlog_timestamp asc,id asc
+        ]]>
+    </select>
+
+    <insert id="insert" parameterClass="metaHistoryDO">
+        insert into meta_history (<include refid="allColumns"/>)
+        values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#sqlSchema#,#sqlTable#,#sqlText#,#sqlType#,#extra#)
+    </insert>
+    
+    <delete id="deleteByName" parameterClass="java.util.Map">
+        delete from meta_history 
+        where destination=#destination#
+    </delete>
+
+
+    <delete id="deleteByGmtModified" parameterClass="java.util.Map">
+        <![CDATA[
+		delete from meta_history
+		where gmt_modified < timestamp(#timestamp#)
+        ]]>
+    </delete>
+</sqlMap>

deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
+<sqlMap namespace="meta_snapshot">
+    <typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO"/>
+    <typeAlias alias="tableMetaSnapshotDO"
+               type="com.alibaba.middleware.jingwei.biz.dataobject.CanalTableMetaSnapshotDO"/>
+    <sql id="allColumns">
+        <![CDATA[
+		gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
+        ]]>
+    </sql>
+    <sql id="allVOColumns">
+        <![CDATA[
+		a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
+		a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
+        ]]>
+    </sql>
+
+    <select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaSnapshotDO">
+    	select <include refid="allVOColumns"/>
+    	<![CDATA[
+        from meta_snapshot a
+        where destination = #destination# and binlog_timestamp < #timestamp#
+        order by binlog_timestamp desc,id desc
+        limit 1
+        ]]>
+    </select>
+    
+    <insert id="insert" parameterClass="metaSnapshotDO">
+        insert into meta_snapshot (<include refid="allColumns"/>)
+        values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#)
+    </insert>
+
+    <update id="update" parameterClass="metaSnapshotDO">
+        update meta_snapshot set gmt_modified=now(),
+        binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra#
+        where destination=#destination# and binlog_timestamp=0
+    </update>
+
+ 	<delete id="deleteByName" parameterClass="java.util.Map">
+        delete from meta_snapshot 
+        where destination=#destination#
+    </delete>
+
+    <delete id="deleteByGmtModified" parameterClass="java.util.Map">
+        <![CDATA[
+		delete from meta_snapshot
+		where gmt_modified < timestamp(#timestamp#)
+        ]]>
+    </delete>
+</sqlMap>

deployer/src/main/resources/spring/tsdb/sql/create_table.sql

@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS `meta_snapshot` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `gmt_create` datetime NOT NULL COMMENT '创建时间',
+  `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+  `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+  `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+  `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+  `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+  `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+  `data` longtext DEFAULT NULL COMMENT '表结构数据',
+  `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+  KEY `destination` (`destination`),
+  KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+  KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表';
+
+CREATE TABLE IF NOT EXISTS `meta_history` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `gmt_create` datetime NOT NULL COMMENT '创建时间',
+  `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+  `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+  `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+  `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+  `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+  `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+  `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
+  `schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
+  `table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
+  `sql` longtext DEFAULT NULL COMMENT '执行的sql',
+  `type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
+  `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+  KEY `destination` (`destination`),
+  KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+  KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表';

driver/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>
@@ -33,10 +33,6 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>org.jboss.netty</groupId>
-			<artifactId>netty</artifactId>
-		</dependency>
 		<!-- test dependency -->
 		<dependency>
 			<groupId>junit</groupId>

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java

@@ -2,19 +2,19 @@ package com.alibaba.otter.canal.parse.driver.mysql;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.SocketChannel;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.ClientAuthenticationPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QuitCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.HandshakeInitializationPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.Reply323Packet;
+import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
+import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannelPool;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
 
@@ -34,17 +34,20 @@ public class MysqlConnector {
     private byte                charsetNumber     = 33;
     private String              defaultSchema     = "retl";
     private int                 soTimeout         = 30 * 1000;
+    private int                 connTimeout       = 5 * 1000;
     private int                 receiveBufferSize = 16 * 1024;
     private int                 sendBufferSize    = 16 * 1024;
 
     private SocketChannel       channel;
+    private volatile boolean    dumping           = false;
+    // mysql connectinnId
+    private long                connectionId      = -1;
     private AtomicBoolean       connected         = new AtomicBoolean(false);
 
     public MysqlConnector(){
     }
 
     public MysqlConnector(InetSocketAddress address, String username, String password){
-
         this.address = address;
         this.username = username;
         this.password = password;
@@ -61,14 +64,12 @@ public class MysqlConnector {
     public void connect() throws IOException {
         if (connected.compareAndSet(false, true)) {
             try {
-                channel = SocketChannel.open();
-                configChannel(channel);
+                channel = SocketChannelPool.open(address);
                 logger.info("connect MysqlConnection to {}...", address);
-                channel.connect(address);
                 negotiate(channel);
             } catch (Exception e) {
                 disconnect();
-                throw new IOException("connect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e));
+                throw new IOException("connect " + this.address + " failure", e);
             }
         } else {
             logger.error("the channel can't be connected twice.");
@@ -86,10 +87,28 @@ public class MysqlConnector {
                 if (channel != null) {
                     channel.close();
                 }
-
                 logger.info("disConnect MysqlConnection to {}...", address);
             } catch (Exception e) {
-                throw new IOException("disconnect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e));
+                throw new IOException("disconnect " + this.address + " failure", e);
+            }
+
+            // 执行一次quit
+            if (dumping && connectionId >= 0) {
+                MysqlConnector connector = null;
+                try {
+                    connector = this.fork();
+                    connector.connect();
+                    MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector);
+                    executor.update("KILL CONNECTION " + connectionId);
+                } catch (Exception e) {
+                    throw new IOException("KILL DUMP " + connectionId + " failure", e);
+                } finally {
+                    if (connector != null) {
+                        connector.disconnect();
+                    }
+                }
+
+                dumping = false;
             }
         } else {
             logger.info("the channel {} is not connected", this.address);
@@ -110,18 +129,18 @@ public class MysqlConnector {
         connector.setReceiveBufferSize(getReceiveBufferSize());
         connector.setSendBufferSize(getSendBufferSize());
         connector.setSoTimeout(getSoTimeout());
+        connector.setConnTimeout(connTimeout);
         return connector;
     }
 
-    // ====================== help method ====================
+    public void quit() throws IOException {
+        QuitCommandPacket quit = new QuitCommandPacket();
+        byte[] cmdBody = quit.toBytes();
 
-    private void configChannel(SocketChannel channel) throws IOException {
-        channel.socket().setKeepAlive(true);
-        channel.socket().setReuseAddress(true);
-        channel.socket().setSoTimeout(soTimeout);
-        channel.socket().setTcpNoDelay(true);
-        channel.socket().setReceiveBufferSize(receiveBufferSize);
-        channel.socket().setSendBufferSize(sendBufferSize);
+        HeaderPacket quitHeader = new HeaderPacket();
+        quitHeader.setPacketBodyLength(cmdBody.length);
+        quitHeader.setPacketSequenceNumber((byte) 0x00);
+        PacketManager.writePkg(channel, quitHeader.toBytes(), cmdBody);
     }
 
     private void negotiate(SocketChannel channel) throws IOException {
@@ -140,6 +159,7 @@ public class MysqlConnector {
         }
         HandshakeInitializationPacket handshakePacket = new HandshakeInitializationPacket();
         handshakePacket.fromBytes(body);
+        connectionId = handshakePacket.threadId; // 记录一下connection
 
         logger.info("handshake initialization packet received, prepare the client authentication packet to send");
 
@@ -157,8 +177,7 @@ public class MysqlConnector {
         h.setPacketBodyLength(clientAuthPkgBody.length);
         h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
 
-        PacketManager.write(channel,
-            new ByteBuffer[] { ByteBuffer.wrap(h.toBytes()), ByteBuffer.wrap(clientAuthPkgBody) });
+        PacketManager.writePkg(channel, h.toBytes(), clientAuthPkgBody);
         logger.info("client authentication packet is sent out.");
 
         // check auth result
@@ -194,7 +213,7 @@ public class MysqlConnector {
         h323.setPacketBodyLength(b323Body.length);
         h323.setPacketSequenceNumber((byte) (packetSequenceNumber + 1));
 
-        PacketManager.write(channel, new ByteBuffer[] { ByteBuffer.wrap(h323.toBytes()), ByteBuffer.wrap(b323Body) });
+        PacketManager.writePkg(channel, h323.toBytes(), b323Body);
         logger.info("client 323 authentication packet is sent out.");
         // check auth result
         HeaderPacket header = PacketManager.readHeader(channel, 4);
@@ -291,4 +310,27 @@ public class MysqlConnector {
         this.password = password;
     }
 
+    public long getConnectionId() {
+        return connectionId;
+    }
+
+    public void setConnectionId(long connectionId) {
+        this.connectionId = connectionId;
+    }
+
+    public boolean isDumping() {
+        return dumping;
+    }
+
+    public void setDumping(boolean dumping) {
+        this.dumping = dumping;
+    }
+
+    public int getConnTimeout() {
+        return connTimeout;
+    }
+
+    public void setConnTimeout(int connTimeout) {
+        this.connTimeout = connTimeout;
+    }
 }

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java

@@ -1,17 +1,18 @@
 package com.alibaba.otter.canal.parse.driver.mysql;
 
 import java.io.IOException;
-import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
 import java.util.List;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.EOFPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetHeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.RowDataPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
 
 /**
@@ -24,9 +25,9 @@ public class MysqlQueryExecutor {
 
     private SocketChannel channel;
 
-    public MysqlQueryExecutor(MysqlConnector connector){
+    public MysqlQueryExecutor(MysqlConnector connector) throws IOException{
         if (!connector.isConnected()) {
-            throw new RuntimeException("should execute connector.connect() first");
+            throw new IOException("should execute connector.connect() first");
         }
 
         this.channel = connector.getChannel();
@@ -51,7 +52,7 @@ public class MysqlQueryExecutor {
         QueryCommandPacket cmd = new QueryCommandPacket();
         cmd.setQueryString(queryString);
         byte[] bodyBytes = cmd.toBytes();
-        PacketManager.write(channel, bodyBytes);
+        PacketManager.writeBody(channel, bodyBytes);
         byte[] body = readNextPacket();
 
         if (body[0] < 0) {
@@ -88,16 +89,70 @@ public class MysqlQueryExecutor {
         for (RowDataPacket r : rowData) {
             resultSet.getFieldValues().addAll(r.getColumns());
         }
-        resultSet.setSourceAddress(channel.socket().getRemoteSocketAddress());
+        resultSet.setSourceAddress(channel.getRemoteSocketAddress());
 
         return resultSet;
     }
 
-    private void readEofPacket() throws IOException {
+    public List<ResultSetPacket> queryMulti(String queryString) throws IOException {
+        QueryCommandPacket cmd = new QueryCommandPacket();
+        cmd.setQueryString(queryString);
+        byte[] bodyBytes = cmd.toBytes();
+        PacketManager.writeBody(channel, bodyBytes);
+        List<ResultSetPacket> resultSets = new ArrayList<ResultSetPacket>();
+        boolean moreResult = true;
+        while (moreResult) {
+            byte[] body = readNextPacket();
+            if (body[0] < 0) {
+                ErrorPacket packet = new ErrorPacket();
+                packet.fromBytes(body);
+                throw new IOException(packet + "\n with command: " + queryString);
+            }
+
+            ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
+            rsHeader.fromBytes(body);
+
+            List<FieldPacket> fields = new ArrayList<FieldPacket>();
+            for (int i = 0; i < rsHeader.getColumnCount(); i++) {
+                FieldPacket fp = new FieldPacket();
+                fp.fromBytes(readNextPacket());
+                fields.add(fp);
+            }
+
+            moreResult = readEofPacket();
+
+            List<RowDataPacket> rowData = new ArrayList<RowDataPacket>();
+            while (true) {
+                body = readNextPacket();
+                if (body[0] == -2) {
+                    break;
+                }
+                RowDataPacket rowDataPacket = new RowDataPacket();
+                rowDataPacket.fromBytes(body);
+                rowData.add(rowDataPacket);
+            }
+
+            ResultSetPacket resultSet = new ResultSetPacket();
+            resultSet.getFieldDescriptors().addAll(fields);
+            for (RowDataPacket r : rowData) {
+                resultSet.getFieldValues().addAll(r.getColumns());
+            }
+            resultSet.setSourceAddress(channel.getRemoteSocketAddress());
+            resultSets.add(resultSet);
+        }
+
+        return resultSets;
+    }
+
+    private boolean readEofPacket() throws IOException {
         byte[] eofBody = readNextPacket();
+        EOFPacket packet = new EOFPacket();
+        packet.fromBytes(eofBody);
         if (eofBody[0] != -2) {
             throw new IOException("EOF Packet is expected, but packet with field_count=" + eofBody[0] + " is found.");
         }
+
+        return (packet.statusFlag & 0x0008) != 0;
     }
 
     protected byte[] readNextPacket() throws IOException {

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java

@@ -1,7 +1,6 @@
 package com.alibaba.otter.canal.parse.driver.mysql;
 
 import java.io.IOException;
-import java.nio.channels.SocketChannel;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,28 +20,29 @@ public class MysqlUpdateExecutor {
 
     private static final Logger logger = LoggerFactory.getLogger(MysqlUpdateExecutor.class);
 
-    private SocketChannel       channel;
+    private MysqlConnector      connector;
 
-    public MysqlUpdateExecutor(MysqlConnector connector){
+    public MysqlUpdateExecutor(MysqlConnector connector) throws IOException{
         if (!connector.isConnected()) {
-            throw new RuntimeException("should execute connector.connect() first");
+            throw new IOException("should execute connector.connect() first");
         }
 
-        this.channel = connector.getChannel();
+        this.connector = connector;
     }
 
-    public MysqlUpdateExecutor(SocketChannel ch){
-        this.channel = ch;
-    }
+    /*
+     * public MysqlUpdateExecutor(SocketChannel ch){ this.channel = ch; }
+     */
 
     public OKPacket update(String updateString) throws IOException {
         QueryCommandPacket cmd = new QueryCommandPacket();
         cmd.setQueryString(updateString);
         byte[] bodyBytes = cmd.toBytes();
-        PacketManager.write(channel, bodyBytes);
+        PacketManager.writeBody(connector.getChannel(), bodyBytes);
 
         logger.debug("read update result...");
-        byte[] body = PacketManager.readBytes(channel, PacketManager.readHeader(channel, 4).getPacketBodyLength());
+        byte[] body = PacketManager.readBytes(connector.getChannel(),
+            PacketManager.readHeader(connector.getChannel(), 4).getPacketBodyLength());
         if (body[0] < 0) {
             ErrorPacket packet = new ErrorPacket();
             packet.fromBytes(body);

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java

@@ -14,15 +14,17 @@ import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
  * </pre>
  * 
  * <br>
- * The Packet Header will not be shown in the descriptions of packets that follow this section. Think of it as always
- * there. But logically, it "precedes the packet" rather than "is included in the packet".<br>
+ * The Packet Header will not be shown in the descriptions of packets that
+ * follow this section. Think of it as always there. But logically, it
+ * "precedes the packet" rather than "is included in the packet".<br>
  * 
  * @author fujohnwang
  */
 public class HeaderPacket implements IPacket {
 
     /**
-     * this field indicates the packet length that follows the header, with header packet's 4 bytes excluded.
+     * this field indicates the packet length that follows the header, with
+     * header packet's 4 bytes excluded.
      */
     private int  packetBodyLength;
     private byte packetSequenceNumber;

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java

@@ -12,6 +12,7 @@ import java.io.IOException;
  * @see 1.0
  */
 public interface IPacket {
+
     /**
      * un-marshal raw bytes into {@link IPacket} state for application usage.<br>
      * 

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java

@@ -47,8 +47,9 @@ public class ClientAuthenticationPacket extends PacketWithHeaderPacket {
         /**
          * CLIENT_LONG_PASSWORD CLIENT_LONG_FLAG CLIENT_PROTOCOL_41
          * CLIENT_INTERACTIVE CLIENT_TRANSACTIONS CLIENT_SECURE_CONNECTION
+         * CLIENT_MULTI_STATEMENTS;
          */
-        ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768, out); // remove
+        ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768 | 0x00010000, out); // remove
         // client_interactive
         // feature
 

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QuitCommandPacket.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
+
+/**
+ * quit cmd
+ * 
+ * @author agapple 2016年3月1日 下午8:33:02
+ * @since 1.0.22
+ */
+public class QuitCommandPacket extends CommandPacket {
+
+    public static final byte[] QUIT = new byte[] { 1, 0, 0, 0, 1 };
+
+    public QuitCommandPacket(){
+        setCommand((byte) 0x01);
+    }
+
+    @Override
+    public void fromBytes(byte[] data) throws IOException {
+
+    }
+
+    @Override
+    public byte[] toBytes() throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        out.write(getCommand());
+        out.write(QUIT);
+        return out.toByteArray();
+    }
+
+}

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java

@@ -74,6 +74,8 @@ public class FieldPacket extends PacketWithHeaderPacket {
         this.decimals = data[index];
         index++;
         //
+        index += 2;// skip filter
+        //
         if (index < data.length) {
             reader.setIndex(index);
             this.definition = reader.readLengthCodedString(data);