fixed default value

Table meta support

.gitignore

@@ -13,3 +13,4 @@ otter.properties
 jtester.properties
 .idea/
 *.iml
+.DS_Store

README.md

@@ -9,7 +9,7 @@
 
 <h1>背景</h1>
 <p style="font-size: 14px;">   早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&amp;消费的业务，从此开启了一段新纪元。</p>
-<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.6及以下的版本(阿里内部mysql 5.6.10, mysql 5.5.18和5.1.40/48)</p>
+<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.7及以下的版本(阿里内部mysql 5.7.13, 5.6.10, mysql 5.5.18和5.1.40/48)</p>
 <p style="font-size: 14px;"> </p>
 <p style="font-size: 14px;">基于日志增量订阅&amp;消费支持的业务：</p>
 <ol style="font-size: 14px;">
@@ -36,7 +36,7 @@
 <li>slave重做中继日志中的事件，将改变反映它自己的数据。</li>
 </ol>
 <h3>canal的工作原理：</h3>
-<p><img width="590" src="http://dl.iteye.com/upload/attachment/0080/3107/c87b67ba-394c-3086-9577-9db05be04c95.jpg" alt="" height="273">
+<p><img width="590" src="https://camo.githubusercontent.com/46c626b4cde399db43b2634a7911a04aecf273a0/687474703a2f2f646c2e69746579652e636f6d2f75706c6f61642f6174746163686d656e742f303038302f333130372f63383762363762612d333934632d333038362d393537372d3964623035626530346339352e6a7067" alt="" height="273">
 <p>原理相对比较简单：</p>
 <ol>
 <li>canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议</li>
@@ -62,6 +62,17 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
 </ul>
 
+<h1>相关资料</h1>
+
+* ADC阿里技术嘉年华分享ppt (放在google docs上，可能需要翻墙): <a href="https://docs.google.com/presentation/d/1MkszUPYRDkfVPz9IqOT1LLT5d9tuwde_WC8GZvjaDRg/edit?usp=sharing">ppt下载</href>  
+* [与阿里巴巴的RocketMQ配合使用](https://github.com/alibaba/RocketMQ)
+
+<h1>相关开源</h1>
+<ol>
+<li>阿里巴巴分布式数据库同步系统(解决中美异地机房)：<a href="http://github.com/alibaba/otter">http://github.com/alibaba/otter</a></li>
+<li>阿里巴巴去Oracle数据迁移同步工具(目标支持MySQL/DRDS)：<a href="http://github.com/alibaba/yugong">http://github.com/alibaba/yugong</a></li>
+</ol>
+
 <h1>问题反馈</h1>
 <ol>
 <li>qq交流群： 161559791 </li>
@@ -69,3 +80,10 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li>新浪微博： agapple0002 </li>
 <li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
 </ol>
+
+<pre>
+【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
+岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
+工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊. 
+具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
+</pre>

client/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java

@@ -19,14 +19,14 @@ public interface CanalConnector {
      * 
      * @throws CanalClientException
      */
-    public void connect() throws CanalClientException;
+    void connect() throws CanalClientException;
 
     /**
      * 释放链接
      * 
      * @throws CanalClientException
      */
-    public void disconnect() throws CanalClientException;
+    void disconnect() throws CanalClientException;
 
     /**
      * 检查下链接是否合法
@@ -43,7 +43,7 @@ public interface CanalConnector {
      * 
      * @throws CanalClientException
      */
-    public boolean checkValid() throws CanalClientException;
+    boolean checkValid() throws CanalClientException;
 
     /**
      * 客户端订阅，重复订阅时会更新对应的filter信息
@@ -56,7 +56,6 @@ public interface CanalConnector {
      * TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
      * </pre>
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void subscribe(String filter) throws CanalClientException;
@@ -64,7 +63,6 @@ public interface CanalConnector {
     /**
      * 客户端订阅，不提交客户端filter，以服务端的filter为准
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void subscribe() throws CanalClientException;
@@ -72,7 +70,6 @@ public interface CanalConnector {
     /**
      * 取消订阅
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void unsubscribe() throws CanalClientException;
@@ -140,14 +137,14 @@ public interface CanalConnector {
     void ack(long batchId) throws CanalClientException;
 
     /**
-     * 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId
+     * 回滚到未进行 {@link #ack} 的地方，指定回滚具体的batchId
      * 
      * @throws CanalClientException
      */
     void rollback(long batchId) throws CanalClientException;
 
     /**
-     * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
+     * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
      * 
      * @throws CanalClientException
      */

client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java

@@ -21,6 +21,7 @@ public class CanalConnectors {
      * 创建单链接的客户端链接
      *
      * @param address
+     * @param destination
      * @param username
      * @param password
      * @return
@@ -36,14 +37,17 @@ public class CanalConnectors {
      * 创建带cluster模式的客户端链接，自动完成failover切换
      *
      * @param addresses
+     * @param destination
      * @param username
      * @param password
      * @return
      */
     public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
                                                      String username, String password) {
-        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
-                                                                         new SimpleNodeAccessStrategy(addresses));
+            password,
+            destination,
+            new SimpleNodeAccessStrategy(addresses));
         canalConnector.setSoTimeout(30 * 1000);
         return canalConnector;
     }
@@ -51,19 +55,18 @@ public class CanalConnectors {
     /**
      * 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描
      *
+     * @param zkServers
+     * @param destination
      * @param username
      * @param password
      * @return
      */
     public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
                                                      String password) {
-        ClusterCanalConnector canalConnector = new ClusterCanalConnector(
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
-                                                                         username,
+            password,
-                                                                         password,
+            destination,
-                                                                         destination,
+            new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkServers)));
-                                                                         new ClusterNodeAccessStrategy(
-                                                                                                       destination,
-                                                                                                       ZkClientx.getZkClient(zkServers)));
         canalConnector.setSoTimeout(30 * 1000);
         return canalConnector;
     }

client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java

@@ -10,5 +10,7 @@ import java.net.SocketAddress;
  */
 public interface CanalNodeAccessStrategy {
 
+    SocketAddress currentNode();
+
     SocketAddress nextNode();
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.client.impl;
 import java.net.SocketAddress;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,11 +40,17 @@ public class ClusterCanalConnector implements CanalConnector {
 
     public void connect() throws CanalClientException {
         while (currentConnector == null) {
-            SocketAddress nextAddress = this.accessStrategy.nextNode();
             int times = 0;
             while (true) {
                 try {
-                    currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination);
+                    currentConnector = new SimpleCanalConnector(null, username, password, destination) {
+
+                        @Override
+                        public SocketAddress getNextAddress() {
+                            return accessStrategy.nextNode();
+                        }
+
+                    };
                     currentConnector.setSoTimeout(soTimeout);
                     if (filter != null) {
                         currentConnector.setFilter(filter);
@@ -57,7 +62,7 @@ public class ClusterCanalConnector implements CanalConnector {
                     currentConnector.connect();
                     break;
                 } catch (Exception e) {
-                    logger.warn("failed to connect to:{} after retry {} times", nextAddress, times);
+                    logger.warn("failed to connect to:{} after retry {} times", accessStrategy.currentNode(), times);
                     currentConnector.disconnect();
                     currentConnector = null;
                     // retry for #retryTimes for each node when trying to
@@ -101,9 +106,10 @@ public class ClusterCanalConnector implements CanalConnector {
                 this.filter = filter;
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when subscribing from server:{}\n{}",
+                logger.warn(String.format(
-                    currentConnector.getAddress(),
+                        "something goes wrong when subscribing from server: %s",
-                    ExceptionUtils.getFullStackTrace(t));
+                        currentConnector != null ? currentConnector.getAddress() : "null"),
+                        t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -120,9 +126,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.unsubscribe();
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when unsubscribing from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -138,9 +143,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.get(batchSize);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getting data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getting data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -156,9 +160,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.get(batchSize, timeout, unit);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getting data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getting data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -174,9 +177,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.getWithoutAck(batchSize);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector.getAddress()), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -192,9 +194,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector.getAddress()), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -210,9 +211,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.rollback(batchId);
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector.getAddress()), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -228,9 +228,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.rollback();
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector.getAddress()), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -247,9 +246,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.ack(batchId);
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when acking data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when acking data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector.getAddress()), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java

@@ -62,6 +62,11 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
         initRunning(this.zkClient.readData(runningPath, true));
     }
 
+    @Override
+    public SocketAddress currentNode() {
+        return nextNode();
+    }
+
     public SocketAddress nextNode() {
         if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点
             return runningAddress;

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java

@@ -5,7 +5,11 @@ import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.SocketChannel;
+import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -37,7 +41,6 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 
 /**
  * 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作
@@ -57,6 +60,8 @@ public class SimpleCanalConnector implements CanalConnector {
     private final ByteBuffer     readHeader            = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
     private final ByteBuffer     writeHeader           = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
     private SocketChannel        channel;
+    private ReadableByteChannel  readableChannel;
+    private WritableByteChannel  writableChannel;
     private List<Compression>    supportedCompressions = new ArrayList<Compression>();
     private ClientIdentity       clientIdentity;
     private ClientRunningMonitor runningMonitor;                                                             // 运行控制
@@ -125,8 +130,14 @@ public class SimpleCanalConnector implements CanalConnector {
         try {
             channel = SocketChannel.open();
             channel.socket().setSoTimeout(soTimeout);
+            SocketAddress address = getAddress();
+            if (address == null) {
+                address = getNextAddress();
+            }
             channel.connect(address);
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            readableChannel = Channels.newChannel(channel.socket().getInputStream());
+            writableChannel = Channels.newChannel(channel.socket().getOutputStream());
+            Packet p = Packet.parseFrom(readNextPacket());
             if (p.getVersion() != 1) {
                 throw new CanalClientException("unsupported version at this client.");
             }
@@ -140,17 +151,17 @@ public class SimpleCanalConnector implements CanalConnector {
             //
             ClientAuth ca = ClientAuth.newBuilder()
                 .setUsername(username != null ? username : "")
+                .setPassword(ByteString.copyFromUtf8(password != null ? password : ""))
                 .setNetReadTimeout(soTimeout)
                 .setNetWriteTimeout(soTimeout)
                 .build();
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.CLIENTAUTHENTICATION)
-                    .setType(PacketType.CLIENTAUTHENTICATION)
+                .setBody(ca.toByteString())
-                    .setBody(ca.toByteString())
+                .build()
-                    .build()
+                .toByteArray());
-                    .toByteArray());
             //
-            Packet ack = Packet.parseFrom(readNextPacket(channel));
+            Packet ack = Packet.parseFrom(readNextPacket());
             if (ack.getType() != PacketType.ACK) {
                 throw new CanalClientException("unexpected packet type when ack is expected");
             }
@@ -169,16 +180,28 @@ public class SimpleCanalConnector implements CanalConnector {
     }
 
     private void doDisconnnect() throws CanalClientException {
+        if (readableChannel != null) {
+            quietlyClose(readableChannel);
+            readableChannel = null;
+        }
+        if (writableChannel != null) {
+            quietlyClose(writableChannel);
+            writableChannel = null;
+        }
         if (channel != null) {
-            try {
+            quietlyClose(channel);
-                channel.close();
-            } catch (IOException e) {
-                logger.warn("exception on closing channel:{} \n {}", channel, e);
-            }
             channel = null;
         }
     }
 
+    private void quietlyClose(Channel channel) {
+        try {
+            channel.close();
+        } catch (IOException e) {
+            logger.warn("exception on closing channel:{} \n {}", channel, e);
+        }
+    }
+
     public void subscribe() throws CanalClientException {
         subscribe(""); // 传递空字符即可
     }
@@ -186,19 +209,18 @@ public class SimpleCanalConnector implements CanalConnector {
     public void subscribe(String filter) throws CanalClientException {
         waitClientRunning();
         try {
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.SUBSCRIPTION)
-                    .setType(PacketType.SUBSCRIPTION)
+                .setBody(Sub.newBuilder()
-                    .setBody(Sub.newBuilder()
+                    .setDestination(clientIdentity.getDestination())
-                        .setDestination(clientIdentity.getDestination())
+                    .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                    .setFilter(filter != null ? filter : "")
-                        .setFilter(filter != null ? filter : "")
-                        .build()
-                        .toByteString())
                     .build()
-                    .toByteArray());
+                    .toByteString())
+                .build()
+                .toByteArray());
             //
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Packet p = Packet.parseFrom(readNextPacket());
             Ack ack = Ack.parseFrom(p.getBody());
             if (ack.getErrorCode() > 0) {
                 throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
@@ -213,18 +235,17 @@ public class SimpleCanalConnector implements CanalConnector {
     public void unsubscribe() throws CanalClientException {
         waitClientRunning();
         try {
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.UNSUBSCRIPTION)
-                    .setType(PacketType.UNSUBSCRIPTION)
+                .setBody(Unsub.newBuilder()
-                    .setBody(Unsub.newBuilder()
+                    .setDestination(clientIdentity.getDestination())
-                        .setDestination(clientIdentity.getDestination())
+                    .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .build()
-                        .toByteString())
                     .build()
-                    .toByteArray());
+                    .toByteString())
+                .build()
+                .toByteArray());
             //
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Packet p = Packet.parseFrom(readNextPacket());
             Ack ack = Ack.parseFrom(p.getBody());
             if (ack.getErrorCode() > 0) {
                 throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
@@ -257,29 +278,27 @@ public class SimpleCanalConnector implements CanalConnector {
                 unit = TimeUnit.MILLISECONDS;
             }
 
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.GET)
-                    .setType(PacketType.GET)
+                .setBody(Get.newBuilder()
-                    .setBody(Get.newBuilder()
+                    .setAutoAck(false)
-                        .setAutoAck(false)
+                    .setDestination(clientIdentity.getDestination())
-                        .setDestination(clientIdentity.getDestination())
+                    .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                    .setFetchSize(size)
-                        .setFetchSize(size)
+                    .setTimeout(time)
-                        .setTimeout(time)
+                    .setUnit(unit.ordinal())
-                        .setUnit(unit.ordinal())
-                        .build()
-                        .toByteString())
                     .build()
-                    .toByteArray());
+                    .toByteString())
-
+                .build()
+                .toByteArray());
             return receiveMessages();
         } catch (IOException e) {
             throw new CanalClientException(e);
         }
     }
 
-    private Message receiveMessages() throws InvalidProtocolBufferException, IOException {
+    private Message receiveMessages() throws IOException {
-        Packet p = Packet.parseFrom(readNextPacket(channel));
+        Packet p = Packet.parseFrom(readNextPacket());
         switch (p.getType()) {
             case MESSAGES: {
                 if (!p.getCompression().equals(Compression.NONE)) {
@@ -311,7 +330,7 @@ public class SimpleCanalConnector implements CanalConnector {
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(channel, Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTACK)
                 .setBody(ca.toByteString())
                 .build()
@@ -329,7 +348,7 @@ public class SimpleCanalConnector implements CanalConnector {
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(channel, Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTROLLBACK)
                 .setBody(ca.toByteString())
                 .build()
@@ -346,7 +365,15 @@ public class SimpleCanalConnector implements CanalConnector {
 
     // ==================== helper method ====================
 
-    private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
+    private void writeWithHeader(byte[] body) throws IOException {
+        writeWithHeader(writableChannel, body);
+    }
+
+    private byte[] readNextPacket() throws IOException {
+        return readNextPacket(readableChannel);
+    }
+
+    private void writeWithHeader(WritableByteChannel channel, byte[] body) throws IOException {
         synchronized (writeDataLock) {
             writeHeader.clear();
             writeHeader.putInt(body.length);
@@ -356,7 +383,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private byte[] readNextPacket(SocketChannel channel) throws IOException {
+    private byte[] readNextPacket(ReadableByteChannel channel) throws IOException {
         synchronized (readDataLock) {
             readHeader.clear();
             read(channel, readHeader);
@@ -367,7 +394,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
+    private void read(ReadableByteChannel channel, ByteBuffer buffer) throws IOException {
         while (buffer.hasRemaining()) {
             int r = channel.read(buffer);
             if (r == -1) {
@@ -411,7 +438,6 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-
     private void waitClientRunning() {
         try {
             if (zkClientx != null) {
@@ -435,6 +461,10 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
+    public SocketAddress getNextAddress() {
+        return null;
+    }
+
     public SocketAddress getAddress() {
         return address;
     }

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java

@@ -32,4 +32,9 @@ public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
         }
     }
 
+    @Override
+    public SocketAddress currentNode() {
+        return nodes.get(index);
+    }
+
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.client.impl.running;
 
 import java.net.InetSocketAddress;
+import java.text.MessageFormat;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -20,6 +21,7 @@ import com.alibaba.otter.canal.common.utils.BooleanMutex;
 import com.alibaba.otter.canal.common.utils.JsonUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 
 /**
  * clinet running控制
@@ -62,6 +64,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
             public void handleDataDeleted(String dataPath) throws Exception {
                 MDC.put("destination", destination);
                 mutex.set(false);
+                // 触发一下退出,可能是人为干预的释放操作或者网络闪断引起的session expired timeout
+                processActiveExit();
                 if (!release && activeData != null && isMine(activeData.getAddress())) {
                     // 如果上一次active的状态就是本机，则即时触发一下active抢占
                     initRunning();
@@ -96,7 +100,11 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
         releaseRunning(); // 尝试一下release
     }
 
-    public void initRunning() {
+    // 改动记录：
+    // 1,在方法上加synchronized关键字,保证同步顺序执行;
+    // 2,判断Zk上已经存在的activeData是否是本机，是的话把mutex重置为true，否则会导致死锁
+    // 3,增加异常处理，保证出现异常时，running节点能被删除,否则会导致死锁
+    public synchronized void initRunning() {
         if (!isStart()) {
             return;
         }
@@ -116,11 +124,22 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
                 initRunning();
             } else {
                 activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+                // 如果发现已经存在,判断一下是否自己,避免活锁
+                if (activeData.getAddress().contains(":") && isMine(activeData.getAddress())) {
+                    mutex.set(true);
+                }
             }
         } catch (ZkNoNodeException e) {
             zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()),
                 true); // 尝试创建父节点
             initRunning();
+        } catch (Throwable t) {
+            logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].",
+                destination),
+                t);
+            // 出现任何异常尝试release
+            releaseRunning();
+            throw new CanalClientException("something goes wrong in initRunning method. ", t);
         }
     }
 

client/src/test/java/logback.xml

@@ -8,7 +8,7 @@
 		</encoder>
 	</appender>
 	
-	<root level="WARN">
+	<root level="INFO">
 		<appender-ref ref="STDOUT"/>
 	</root>
 </configuration>

codeformat.xml

@@ -0,0 +1,291 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<profiles version="12">
+<profile kind="CodeFormatterProfile" name="canal-format" version="12">
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
+<setting id="org.eclipse.jdt.core.compiler.source" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="120"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="true"/>
+</profile>
+</profiles>

codetemplates.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><templates><template autoinsert="true" context="gettercomment_context" deleted="false" description="Comment for getter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name="gettercomment">/**
+ * @return the ${bare_field_name}
+ */</template><template autoinsert="true" context="settercomment_context" deleted="false" description="Comment for setter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.settercomment" name="settercomment">/**
+ * @param ${param} the ${bare_field_name} to set
+ */</template><template autoinsert="true" context="constructorcomment_context" deleted="false" description="Comment for created constructors" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name="constructorcomment">/**
+ * ${tags}
+ */</template><template autoinsert="false" context="filecomment_context" deleted="false" description="Comment for created Java files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.filecomment" name="filecomment"/><template autoinsert="false" context="typecomment_context" deleted="false" description="Comment for created types" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.typecomment" name="typecomment">/**
+ * @author ${user} ${date} ${time}
+ * @since 1.0.0
+ */</template><template autoinsert="true" context="fieldcomment_context" deleted="false" description="Comment for fields" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name="fieldcomment">/**
+ * 
+ */</template><template autoinsert="true" context="methodcomment_context" deleted="false" description="Comment for non-overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name="methodcomment">/**
+ * ${tags}
+ */</template><template autoinsert="false" context="overridecomment_context" deleted="false" description="Comment for overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name="overridecomment"/><template autoinsert="true" context="delegatecomment_context" deleted="false" description="Comment for delegate methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name="delegatecomment">/**
+ * ${tags}
+ * ${see_to_target}
+ */</template><template autoinsert="false" context="newtype_context" deleted="false" description="Newly created files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.newtype" name="newtype">${filecomment}
+${package_declaration}
+${typecomment}
+${type_declaration}</template><template autoinsert="true" context="classbody_context" deleted="false" description="Code in new class type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.classbody" name="classbody">
+</template><template autoinsert="true" context="interfacebody_context" deleted="false" description="Code in new interface type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name="interfacebody">
+</template><template autoinsert="true" context="enumbody_context" deleted="false" description="Code in new enum type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.enumbody" name="enumbody">
+</template><template autoinsert="true" context="annotationbody_context" deleted="false" description="Code in new annotation type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name="annotationbody">
+</template><template autoinsert="true" context="catchblock_context" deleted="false" description="Code in new catch blocks" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.catchblock" name="catchblock">// ${todo} Auto-generated catch block
+${exception_var}.printStackTrace();</template><template autoinsert="true" context="methodbody_context" deleted="false" description="Code in created method stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodbody" name="methodbody">// ${todo} Auto-generated method stub
+${body_statement}</template><template autoinsert="true" context="constructorbody_context" deleted="false" description="Code in created constructor stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name="constructorbody">${body_statement}
+// ${todo} Auto-generated constructor stub</template><template autoinsert="true" context="getterbody_context" deleted="false" description="Code in created getters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.getterbody" name="getterbody">return ${field};</template><template autoinsert="true" context="setterbody_context" deleted="false" description="Code in created setters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.setterbody" name="setterbody">${field} = ${param};</template></templates>

common/pom.xml

@@ -1,24 +1,29 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
-	<groupId>com.alibaba.otter</groupId>
 	<artifactId>canal.common</artifactId>
 	<packaging>jar</packaging>
 	<name>canal common module for otter ${project.version}</name>
 	<url>http://b2b-doc.alibaba-inc.com/display/opentech/Otter</url>
 	<dependencies>
+		<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
+		<dependency>
+			<groupId>io.netty</groupId>
+			<artifactId>netty-all</artifactId>
+		</dependency>
 		<!-- zk -->
 		<dependency>
 			<groupId>org.apache.zookeeper</groupId>
 			<artifactId>zookeeper</artifactId>
 		</dependency>
 		<dependency>
-			<groupId>com.github.sgroschupf</groupId>
+			<groupId>com.101tec</groupId>
 			<artifactId>zkclient</artifactId>
 		</dependency>
 		<!-- external -->
@@ -29,11 +34,10 @@
 		<dependency>
 			<groupId>commons-lang</groupId>
 			<artifactId>commons-lang</artifactId>
-			<version>2.6</version>
 		</dependency>
 		<dependency>
-			<groupId>org.springframework</groupId>
+			<groupId>commons-codec</groupId>
-			<artifactId>spring</artifactId>
+			<artifactId>commons-codec</artifactId>
 		</dependency>
 		<dependency>
 			<groupId>com.alibaba</groupId>
@@ -60,10 +64,37 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-aop</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-context</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-jdbc</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-orm</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-test</artifactId>
+			<version>${spring_version}</version>
+			<scope>test</scope>
+		</dependency>
 		<!-- junit -->
 		<dependency>
-		  <groupId>junit</groupId>
+			<groupId>junit</groupId>
-		  <artifactId>junit</artifactId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
 		</dependency>
-  </dependencies>
+	</dependencies>
 </project>

common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java

@@ -6,9 +6,9 @@ package com.alibaba.otter.canal.common;
  */
 public interface CanalLifeCycle {
 
-    public void start();
+    void start();
 
-    public void stop();
+    void stop();
 
-    public boolean isStart();
+    boolean isStart();
 }

common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java

@@ -14,8 +14,7 @@ public interface CanalAlarmHandler extends CanalLifeCycle {
      * 发送对应destination的报警
      * 
      * @param destination
-     * @param title
      * @param msg
      */
-    public void sendAlarm(String destination, String msg);
+    void sendAlarm(String destination, String msg);
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java

@@ -38,7 +38,8 @@ public class AddressUtils {
     private static boolean isValidHostAddress(InetAddress address) {
         if (address == null || address.isLoopbackAddress()) return false;
         String name = address.getHostAddress();
-        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches());
+        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name)
+            .matches());
     }
 
     public static String getHostIp() {

common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java

@@ -1,12 +1,19 @@
 package com.alibaba.otter.canal.common.utils;
 
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
 import java.util.Arrays;
 import java.util.List;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.TypeReference;
 import com.alibaba.fastjson.serializer.JSONSerializer;
+import com.alibaba.fastjson.serializer.ObjectSerializer;
 import com.alibaba.fastjson.serializer.PropertyFilter;
+import com.alibaba.fastjson.serializer.SerializeConfig;
 import com.alibaba.fastjson.serializer.SerializeWriter;
 import com.alibaba.fastjson.serializer.SerializerFeature;
 
@@ -17,6 +24,12 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
  */
 public class JsonUtils {
 
+    static {
+        SerializeConfig.getGlobalInstance().put(InetAddress.class, InetAddressSerializer.instance);
+        SerializeConfig.getGlobalInstance().put(Inet4Address.class, InetAddressSerializer.instance);
+        SerializeConfig.getGlobalInstance().put(Inet6Address.class, InetAddressSerializer.instance);
+    }
+
     public static <T> T unmarshalFromByte(byte[] bytes, Class<T> targetClass) {
         return (T) JSON.parseObject(bytes, targetClass);// 默认为UTF-8
     }
@@ -70,4 +83,22 @@ public class JsonUtils {
             out.close();
         }
     }
+
+    public static class InetAddressSerializer implements ObjectSerializer {
+
+        public static InetAddressSerializer instance = new InetAddressSerializer();
+
+        @Override
+        public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features)
+                                                                                                                   throws IOException {
+            if (object == null) {
+                serializer.writeNull();
+                return;
+            }
+
+            InetAddress address = (InetAddress) object;
+            // 优先使用name
+            serializer.write(address.getHostName());
+        }
+    }
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java

@@ -54,6 +54,7 @@ public final class UriUtils {
         }
         String query = uri.getRawQuery();
         HashMap<String, String> params = new HashMap<String, String>();
+        @SuppressWarnings("resource")
         Scanner scan = new Scanner(query);
         scan.useDelimiter(SPLIT);
         while (scan.hasNext()) {

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java

@@ -12,7 +12,7 @@ import org.I0Itec.zkclient.serialize.ZkSerializer;
 import org.apache.zookeeper.CreateMode;
 
 import com.google.common.base.Function;
-import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * 使用自定义的ZooKeeperx for zk connection
@@ -23,7 +23,7 @@ import com.google.common.collect.MapMaker;
 public class ZkClientx extends ZkClient {
 
     // 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
-    private static Map<String, ZkClientx> clients = new MapMaker().makeComputingMap(new Function<String, ZkClientx>() {
+    private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>() {
 
                                                       public ZkClientx apply(String servers) {
                                                           return new ZkClientx(servers);
@@ -62,10 +62,12 @@ public class ZkClientx extends ZkClient {
      * Create a persistent Sequential node.
      * 
      * @param path
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * @param createParents if true all parent dirs are created as well and no
-     * in case the path already exists
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws ZkInterruptedException if operation was interrupted, or a
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */
@@ -89,10 +91,12 @@ public class ZkClientx extends ZkClient {
      * 
      * @param path
      * @param data
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * @param createParents if true all parent dirs are created as well and no
-     * in case the path already exists
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws ZkInterruptedException if operation was interrupted, or a
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */
@@ -118,10 +122,12 @@ public class ZkClientx extends ZkClient {
      * 
      * @param path
      * @param data
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * @param createParents if true all parent dirs are created as well and no
-     * in case the path already exists
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws ZkInterruptedException if operation was interrupted, or a
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java

@@ -8,20 +8,15 @@ import java.util.List;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.I0Itec.zkclient.IZkConnection;
+import org.I0Itec.zkclient.ZkConnection;
 import org.I0Itec.zkclient.exception.ZkException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zookeeper.ClientCnxn;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.client.ConnectStringParser;
 import org.apache.zookeeper.client.HostProvider;
 import org.apache.zookeeper.client.StaticHostProvider;
-import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.ReflectionUtils;
@@ -32,20 +27,20 @@ import org.springframework.util.ReflectionUtils;
  * @author jianghang 2012-7-10 下午02:31:42
  * @version 1.0.0
  */
-public class ZooKeeperx implements IZkConnection {
+public class ZooKeeperx extends ZkConnection {
 
     private static final String SERVER_COMMA            = ";";
     private static final Logger logger                  = LoggerFactory.getLogger(ZooKeeperx.class);
     private static final Field  clientCnxnField         = ReflectionUtils.findField(ZooKeeper.class, "cnxn");
     private static final Field  hostProviderField       = ReflectionUtils.findField(ClientCnxn.class, "hostProvider");
     private static final Field  serverAddressesField    = ReflectionUtils.findField(StaticHostProvider.class,
-                                                                                    "serverAddresses");
+                                                            "serverAddresses");
+    private static final Field  zookeeperLockField      = ReflectionUtils.findField(ZkConnection.class,
+                                                            "_zookeeperLock");
+    private static final Field  zookeeperFiled          = ReflectionUtils.findField(ZkConnection.class, "_zk");
     private static final int    DEFAULT_SESSION_TIMEOUT = 90000;
 
-    private ZooKeeper           _zk                     = null;
+    private final List<String>  _serversList;
-    private Lock                _zookeeperLock          = new ReentrantLock();
-
-    private final List<String>  _servers;
     private final int           _sessionTimeOut;
 
     public ZooKeeperx(String zkServers){
@@ -53,115 +48,60 @@ public class ZooKeeperx implements IZkConnection {
     }
 
     public ZooKeeperx(String zkServers, int sessionTimeOut){
-        _servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA));
+        super(zkServers, sessionTimeOut);
+        _serversList = Arrays.asList(StringUtils.split(this.getServers(), SERVER_COMMA));
         _sessionTimeOut = sessionTimeOut;
     }
 
     @Override
     public void connect(Watcher watcher) {
+        ReflectionUtils.makeAccessible(zookeeperLockField);
+        ReflectionUtils.makeAccessible(zookeeperFiled);
+        Lock _zookeeperLock = (ReentrantLock) ReflectionUtils.getField(zookeeperLockField, this);
+        ZooKeeper _zk = (ZooKeeper) ReflectionUtils.getField(zookeeperFiled, this);
+
         _zookeeperLock.lock();
         try {
             if (_zk != null) {
                 throw new IllegalStateException("zk client has already been started");
             }
+            String zkServers = _serversList.get(0);
 
             try {
-                logger.debug("Creating new ZookKeeper instance to connect to " + _servers + ".");
+                logger.debug("Creating new ZookKeeper instance to connect to " + zkServers + ".");
-                _zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher);
+                _zk = new ZooKeeper(zkServers, _sessionTimeOut, watcher);
                 configMutliCluster(_zk);
+                ReflectionUtils.setField(zookeeperFiled, this, _zk);
             } catch (IOException e) {
-                throw new ZkException("Unable to connect to " + _servers, e);
+                throw new ZkException("Unable to connect to " + zkServers, e);
             }
         } finally {
             _zookeeperLock.unlock();
         }
     }
 
-    public void close() throws InterruptedException {
-        _zookeeperLock.lock();
-        try {
-            if (_zk != null) {
-                logger.debug("Closing ZooKeeper connected to " + _servers);
-                _zk.close();
-                _zk = null;
-            }
-        } finally {
-            _zookeeperLock.unlock();
-        }
-    }
-
-    public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException {
-        return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode);
-    }
-
-    public void delete(String path) throws InterruptedException, KeeperException {
-        _zk.delete(path, -1);
-    }
-
-    public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException {
-        return _zk.exists(path, watch) != null;
-    }
-
-    public List<String> getChildren(final String path, final boolean watch) throws KeeperException,
-                                                                           InterruptedException {
-        return _zk.getChildren(path, watch);
-    }
-
-    public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException {
-        return _zk.getData(path, watch, stat);
-    }
-
-    public void writeData(String path, byte[] data) throws KeeperException, InterruptedException {
-        writeData(path, data, -1);
-    }
-
-    public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException {
-        _zk.setData(path, data, version);
-    }
-
-    public States getZookeeperState() {
-        return _zk != null ? _zk.getState() : null;
-    }
-
-    public ZooKeeper getZookeeper() {
-        return _zk;
-    }
-
-    public long getCreateTime(String path) throws KeeperException, InterruptedException {
-        Stat stat = _zk.exists(path, false);
-        if (stat != null) {
-            return stat.getCtime();
-        }
-        return -1;
-    }
-
-    public String getServers() {
-        return StringUtils.join(_servers, SERVER_COMMA);
-    }
-
     // ===============================
 
     public void configMutliCluster(ZooKeeper zk) {
-        if (_servers.size() == 1) {
+        if (_serversList.size() == 1) {
             return;
         }
-        String cluster1 = _servers.get(0);
+        String cluster1 = _serversList.get(0);
         try {
-            if (_servers.size() > 1) {
+            if (_serversList.size() > 1) {
                 // 强制的声明accessible
                 ReflectionUtils.makeAccessible(clientCnxnField);
                 ReflectionUtils.makeAccessible(hostProviderField);
                 ReflectionUtils.makeAccessible(serverAddressesField);
 
                 // 添加第二组集群列表
-                for (int i = 1; i < _servers.size(); i++) {
+                for (int i = 1; i < _serversList.size(); i++) {
-                    String cluster = _servers.get(i);
+                    String cluster = _serversList.get(i);
                     // 强制获取zk中的地址信息
                     ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
                     HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
-                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(
+                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(serverAddressesField,
-                                                                                                             serverAddressesField,
+                        hostProvider);
-                                                                                                             hostProvider);
                     // 添加第二组集群列表
                     serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());
                 }

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java

@@ -120,8 +120,10 @@ public class ZookeeperPathUtils {
     }
 
     public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) {
-        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName,
+        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH,
-                                    String.valueOf(clientId), getBatchMarkNode(batchId));
+            destinationName,
+            String.valueOf(clientId),
+            getBatchMarkNode(batchId));
     }
 
     public static String getCursorPath(String destination, short clientId) {

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java

@@ -87,6 +87,10 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
 
     }
 
+    public void init() {
+        processStart();
+    }
+
     public void start() {
         super.start();
         processStart();

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java

@@ -11,7 +11,8 @@ import java.util.Map;
 public class ServerRunningMonitors {
 
     private static ServerRunningData serverData;
-    private static Map               runningMonitors; // <String, ServerRunningMonitor>
+    private static Map               runningMonitors; // <String,
+                                                      // ServerRunningMonitor>
 
     public static ServerRunningData getServerData() {
         return serverData;

common/src/main/java/com/google/common/collect/MigrateMap.java

@@ -0,0 +1,19 @@
+package com.google.common.collect;
+
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.base.Function;
+
+public class MigrateMap {
+
+    @SuppressWarnings("deprecation")
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker,
+                                                              Function<? super K, ? extends V> computingFunction) {
+        return maker.makeComputingMap(computingFunction);
+    }
+
+    @SuppressWarnings("deprecation")
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
+        return new MapMaker().makeComputingMap(computingFunction);
+    }
+}

dbsync/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java

@@ -6,24 +6,24 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * An utility class implements MySQL/Java charsets conversion.  you can see com.mysql.jdbc.CharsetMapping.
+ * An utility class implements MySQL/Java charsets conversion. you can see
+ * com.mysql.jdbc.CharsetMapping.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  */
-public final class CharsetConversion
+public final class CharsetConversion {
-{
+
     static final Log logger = LogFactory.getLog(CharsetConversion.class);
 
-    static final class Entry
+    static final class Entry {
-    {
+
         protected final int    charsetId;
         protected final String mysqlCharset;
         protected final String mysqlCollation;
         protected final String javaCharset;
 
         Entry(final int id, String mysqlCharset, // NL
-                String mysqlCollation, String javaCharset)
+              String mysqlCollation, String javaCharset){
-        {
             this.charsetId = id;
             this.mysqlCharset = mysqlCharset;
             this.mysqlCollation = mysqlCollation;
@@ -34,38 +34,31 @@ public final class CharsetConversion
     // Character set data used in lookups. The array will be sparse.
     static final Entry[] entries = new Entry[0xff];
 
-    static Entry getEntry(final int id)
+    static Entry getEntry(final int id) {
-    {
+        if (id >= 0 && id < entries.length) {
-        if (id >= 0 && id < entries.length)
-        {
             return entries[id];
-        }
+        } else {
-        else
-        {
             throw new IllegalArgumentException("Invalid charset id: " + id);
         }
     }
 
     // Loads character set information.
-    static void putEntry(final int charsetId, String mysqlCharset,
+    static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation, String javaCharset) {
-            String mysqlCollation, String javaCharset)
-    {
         entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
-                mysqlCollation, javaCharset);
+            mysqlCollation,
+            javaCharset);
     }
 
     // Loads character set information.
     @Deprecated
-    static void putEntry(final int charsetId, String mysqlCharset,
+    static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation) {
-            String mysqlCollation)
-    {
         entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
-                mysqlCollation, /* Unknown java charset */null);
+            mysqlCollation, /* Unknown java charset */
+            null);
     }
 
     // Load character set data statically.
-    static
+    static {
-    {
         putEntry(1, "big5", "big5_chinese_ci", "Big5");
         putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
         putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
@@ -135,7 +128,7 @@ public final class CharsetConversion
         putEntry(69, "dec8", "dec8_bin", "US-ASCII");
         putEntry(70, "greek", "greek_bin", "ISO8859_7");
         putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
-        putEntry(72, "hp8", "hp8_bin" , "US-ASCII");
+        putEntry(72, "hp8", "hp8_bin", "US-ASCII");
         putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
         putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
         putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
@@ -274,16 +267,12 @@ public final class CharsetConversion
     /**
      * Return defined charset name for mysql.
      */
-    public static String getCharset(final int id)
+    public static String getCharset(final int id) {
-    {
         Entry entry = getEntry(id);
 
-        if (entry != null)
+        if (entry != null) {
-        {
             return entry.mysqlCharset;
-        }
+        } else {
-        else
-        {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
@@ -292,16 +281,12 @@ public final class CharsetConversion
     /**
      * Return defined collaction name for mysql.
      */
-    public static String getCollation(final int id)
+    public static String getCollation(final int id) {
-    {
         Entry entry = getEntry(id);
 
-        if (entry != null)
+        if (entry != null) {
-        {
             return entry.mysqlCollation;
-        }
+        } else {
-        else
-        {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
@@ -310,55 +295,41 @@ public final class CharsetConversion
     /**
      * Return converted charset name for java.
      */
-    public static String getJavaCharset(final int id)
+    public static String getJavaCharset(final int id) {
-    {
         Entry entry = getEntry(id);
 
-        if (entry != null)
+        if (entry != null) {
-        {
+            if (entry.javaCharset != null) {
-            if (entry.javaCharset != null)
-            {
                 return entry.javaCharset;
-            }
+            } else {
-            else
+                logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
-            {
+                            + entry.mysqlCollation);
-                logger.warn("Unknown java charset for: id = " + id
-                        + ", name = " + entry.mysqlCharset + ", coll = "
-                        + entry.mysqlCollation);
                 return null;
             }
-        }
+        } else {
-        else
-        {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
     }
 
-    public static void main(String[] args)
+    public static void main(String[] args) {
-    {
+        for (int i = 0; i < entries.length; i++) {
-        for (int i = 0; i < entries.length; i++)
-        {
             Entry entry = entries[i];
 
             System.out.print(i);
             System.out.print(',');
             System.out.print(' ');
-            if (entry != null)
+            if (entry != null) {
-            {
                 System.out.print(entry.mysqlCharset);
                 System.out.print(',');
                 System.out.print(' ');
                 System.out.print(entry.javaCharset);
-                if (entry.javaCharset != null)
+                if (entry.javaCharset != null) {
-                {
                     System.out.print(',');
                     System.out.print(' ');
                     System.out.print(Charset.forName(entry.javaCharset).name());
                 }
-            }
+            } else {
-            else
-            {
                 System.out.print("null");
             }
             System.out.println();

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java

@@ -21,16 +21,13 @@ import org.apache.commons.logging.LogFactory;
  * DirectLogFetcher fetcher = new DirectLogFetcher();
  * fetcher.open(conn, file, 0, 13);
  * 
- * while (fetcher.fetch())
+ * while (fetcher.fetch()) {
- * {
  *     LogEvent event;
- *     do
+ *     do {
- *     {
  *         event = decoder.decode(fetcher, context);
  * 
  *         // process log event.
- *     }
+ *     } while (event != null);
- *     while (event != null);
  * }
  * // connection closed.
  * </pre>
@@ -38,97 +35,79 @@ import org.apache.commons.logging.LogFactory;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DirectLogFetcher extends LogFetcher
+public final class DirectLogFetcher extends LogFetcher {
-{
+
-    protected static final Log logger                = LogFactory.getLog(DirectLogFetcher.class);
+    protected static final Log logger                          = LogFactory.getLog(DirectLogFetcher.class);
 
     /** Command to dump binlog */
-    public static final byte   COM_BINLOG_DUMP       = 18;
+    public static final byte   COM_BINLOG_DUMP                 = 18;
 
     /** Packet header sizes */
-    public static final int    NET_HEADER_SIZE       = 4;
+    public static final int    NET_HEADER_SIZE                 = 4;
-    public static final int    SQLSTATE_LENGTH       = 5;
+    public static final int    SQLSTATE_LENGTH                 = 5;
 
     /** Packet offsets */
-    public static final int    PACKET_LEN_OFFSET     = 0;
+    public static final int    PACKET_LEN_OFFSET               = 0;
-    public static final int    PACKET_SEQ_OFFSET     = 3;
+    public static final int    PACKET_SEQ_OFFSET               = 3;
 
     /** Maximum packet length */
-    public static final int    MAX_PACKET_LENGTH     = (256 * 256 * 256 - 1);
+    public static final int    MAX_PACKET_LENGTH               = (256 * 256 * 256 - 1);
 
     /** BINLOG_DUMP options */
-    public static final int    BINLOG_DUMP_NON_BLOCK = 1;
+    public static final int    BINLOG_DUMP_NON_BLOCK           = 1;
     public static final int    BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
 
     private Connection         conn;
     private OutputStream       mysqlOutput;
     private InputStream        mysqlInput;
 
-    public DirectLogFetcher()
+    public DirectLogFetcher(){
-    {
         super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
     }
 
-    public DirectLogFetcher(final int initialCapacity)
+    public DirectLogFetcher(final int initialCapacity){
-    {
         super(initialCapacity, DEFAULT_GROWTH_FACTOR);
     }
 
-    public DirectLogFetcher(final int initialCapacity, final float growthFactor)
+    public DirectLogFetcher(final int initialCapacity, final float growthFactor){
-    {
         super(initialCapacity, growthFactor);
     }
 
-    private static final Object unwrapConnection(Object conn, Class<?> connClazz)
+    private static final Object unwrapConnection(Object conn, Class<?> connClazz) throws IOException {
-            throws IOException
+        while (!connClazz.isInstance(conn)) {
-    {
+            try {
-        while (!connClazz.isInstance(conn))
-        {
-            try
-            {
                 Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
-                if (connProxy.isInstance(conn))
+                if (connProxy.isInstance(conn)) {
-                {
                     conn = invokeMethod(conn, connProxy, "getTargetConnection");
                     continue;
                 }
-            }
+            } catch (ClassNotFoundException e) {
-            catch (ClassNotFoundException e)
+                // org.springframework.jdbc.datasource.ConnectionProxy not
-            {
+                // found.
-                // org.springframework.jdbc.datasource.ConnectionProxy not found.
             }
 
-            try
+            try {
-            {
                 Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
-                if (connProxy.isInstance(conn))
+                if (connProxy.isInstance(conn)) {
-                {
                     conn = getDeclaredField(conn, connProxy, "_conn");
                     continue;
                 }
-            }
+            } catch (ClassNotFoundException e) {
-            catch (ClassNotFoundException e)
-            {
                 // org.apache.commons.dbcp.DelegatingConnection not found.
             }
 
-            try
+            try {
-            {
+                if (conn instanceof java.sql.Wrapper) {
-                if (conn instanceof java.sql.Wrapper)
-                {
                     Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
                     conn = ((java.sql.Wrapper) conn).unwrap(connIface);
                     continue;
                 }
-            }
+            } catch (ClassNotFoundException e) {
-            catch (ClassNotFoundException e)
-            {
                 // com.mysql.jdbc.Connection not found.
-            }
+            } catch (SQLException e) {
-            catch (SQLException e)
+                logger.warn("Unwrap " + conn.getClass().getName() + " to " + connClazz.getName() + " failed: "
-            {
+                            + e.getMessage(),
-                logger.warn("Unwrap " + conn.getClass().getName() + " to "
+                    e);
-                        + connClazz.getName() + " failed: " + e.getMessage(), e);
             }
 
             return null;
@@ -136,127 +115,85 @@ public final class DirectLogFetcher extends LogFetcher
         return conn;
     }
 
-    private static final Object invokeMethod(Object obj, Class<?> objClazz,
+    private static final Object invokeMethod(Object obj, Class<?> objClazz, String name) {
-            String name)
+        try {
-    {
-        try
-        {
             Method method = objClazz.getMethod(name, (Class<?>[]) null);
             return method.invoke(obj, (Object[]) null);
-        }
+        } catch (NoSuchMethodException e) {
-        catch (NoSuchMethodException e)
+            throw new IllegalArgumentException("No such method: \'" + name + "\' @ " + objClazz.getName(), e);
-        {
+        } catch (IllegalAccessException e) {
-            throw new IllegalArgumentException("No such method: \'" + name
+            throw new IllegalArgumentException("Cannot invoke method: \'" + name + "\' @ " + objClazz.getName(), e);
-                    + "\' @ " + objClazz.getName(), e);
+        } catch (InvocationTargetException e) {
-        }
+            throw new IllegalArgumentException("Invoke method failed: \'" + name + "\' @ " + objClazz.getName(),
-        catch (IllegalAccessException e)
+                e.getTargetException());
-        {
-            throw new IllegalArgumentException("Cannot invoke method: \'"
-                    + name + "\' @ " + objClazz.getName(), e);
-        }
-        catch (InvocationTargetException e)
-        {
-            throw new IllegalArgumentException("Invoke method failed: \'"
-                    + name + "\' @ " + objClazz.getName(),
-                    e.getTargetException());
         }
     }
 
-    private static final Object getDeclaredField(Object obj, Class<?> objClazz,
+    private static final Object getDeclaredField(Object obj, Class<?> objClazz, String name) {
-            String name)
+        try {
-    {
-        try
-        {
             Field field = objClazz.getDeclaredField(name);
             field.setAccessible(true);
             return field.get(obj);
-        }
+        } catch (NoSuchFieldException e) {
-        catch (NoSuchFieldException e)
+            throw new IllegalArgumentException("No such field: \'" + name + "\' @ " + objClazz.getName(), e);
-        {
+        } catch (IllegalAccessException e) {
-            throw new IllegalArgumentException("No such field: \'" + name
+            throw new IllegalArgumentException("Cannot get field: \'" + name + "\' @ " + objClazz.getName(), e);
-                    + "\' @ " + objClazz.getName(), e);
-        }
-        catch (IllegalAccessException e)
-        {
-            throw new IllegalArgumentException("Cannot get field: \'" + name
-                    + "\' @ " + objClazz.getName(), e);
         }
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final int serverId)
+    public void open(Connection conn, String fileName, final int serverId) throws IOException {
-            throws IOException
-    {
         open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final int serverId,
+    public void open(Connection conn, String fileName, final int serverId, boolean nonBlocking) throws IOException {
-            boolean nonBlocking) throws IOException
-    {
         open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final long filePosition,
+    public void open(Connection conn, String fileName, final long filePosition, final int serverId) throws IOException {
-            final int serverId) throws IOException
-    {
         open(conn, fileName, filePosition, serverId, false);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, long filePosition,
+    public void open(Connection conn, String fileName, long filePosition, final int serverId, boolean nonBlocking)
-            final int serverId, boolean nonBlocking) throws IOException
+                                                                                                                  throws IOException {
-    {
+        try {
-        try
-        {
             this.conn = conn;
             Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
             Object unwrapConn = unwrapConnection(conn, connClazz);
-            if (unwrapConn == null)
+            if (unwrapConn == null) {
-            {
+                throw new IOException("Unable to unwrap " + conn.getClass().getName()
-                throw new IOException("Unable to unwrap "
+                                      + " to com.mysql.jdbc.ConnectionImpl");
-                        + conn.getClass().getName()
-                        + " to com.mysql.jdbc.ConnectionImpl");
             }
 
             // Get underlying IO streams for network communications.
             Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
-            if (connIo == null)
+            if (connIo == null) {
-            {
+                throw new IOException("Get null field:" + conn.getClass().getName() + "#io");
-                throw new IOException("Get null field:"
-                        + conn.getClass().getName() + "#io");
             }
-            mysqlOutput = (OutputStream) getDeclaredField(connIo,
+            mysqlOutput = (OutputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlOutput");
-                    connIo.getClass(), "mysqlOutput");
+            mysqlInput = (InputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlInput");
-            mysqlInput = (InputStream) getDeclaredField(connIo,
-                    connIo.getClass(), "mysqlInput");
 
-            if (filePosition == 0)
+            if (filePosition == 0) filePosition = BIN_LOG_HEADER_SIZE;
-                filePosition = BIN_LOG_HEADER_SIZE;
             sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
             position = 0;
-        }
+        } catch (IOException e) {
-        catch (IOException e)
-        {
             close(); /* Do cleanup */
-            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
+            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName + ", position = " + filePosition);
-                    + ", position = " + filePosition);
             throw e;
-        }
+        } catch (ClassNotFoundException e) {
-        catch (ClassNotFoundException e)
-        {
             close(); /* Do cleanup */
-            throw new IOException(
+            throw new IOException("Unable to load com.mysql.jdbc.ConnectionImpl", e);
-                    "Unable to load com.mysql.jdbc.ConnectionImpl", e);
         }
     }
 
@@ -265,8 +202,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param b the byte to put in the buffer
      */
-    protected final void putByte(byte b)
+    protected final void putByte(byte b) {
-    {
         ensureCapacity(position + 1);
 
         buffer[position++] = b;
@@ -277,8 +213,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param i16 the integer to put in the buffer
      */
-    protected final void putInt16(int i16)
+    protected final void putInt16(int i16) {
-    {
         ensureCapacity(position + 2);
 
         byte[] buf = buffer;
@@ -291,8 +226,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param i32 the integer to put in the buffer
      */
-    protected final void putInt32(long i32)
+    protected final void putInt32(long i32) {
-    {
         ensureCapacity(position + 4);
 
         byte[] buf = buffer;
@@ -307,8 +241,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param s the value to put in the buffer
      */
-    protected final void putString(String s)
+    protected final void putString(String s) {
-    {
         ensureCapacity(position + (s.length() * 2) + 1);
 
         System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
@@ -316,10 +249,8 @@ public final class DirectLogFetcher extends LogFetcher
         buffer[position++] = 0;
     }
 
-    protected final void sendBinlogDump(String fileName,
+    protected final void sendBinlogDump(String fileName, final long filePosition, final int serverId,
-            final long filePosition, final int serverId, boolean nonBlocking)
+                                        boolean nonBlocking) throws IOException {
-            throws IOException
-    {
         position = NET_HEADER_SIZE;
 
         putByte(COM_BINLOG_DUMP);
@@ -345,13 +276,10 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
      */
-    public boolean fetch() throws IOException
+    public boolean fetch() throws IOException {
-    {
+        try {
-        try
-        {
             // Fetching packet header from input.
-            if (!fetch0(0, NET_HEADER_SIZE))
+            if (!fetch0(0, NET_HEADER_SIZE)) {
-            {
                 logger.warn("Reached end of input stream while fetching header");
                 return false;
             }
@@ -359,61 +287,48 @@ public final class DirectLogFetcher extends LogFetcher
             // Fetching the first packet(may a multi-packet).
             int netlen = getUint24(PACKET_LEN_OFFSET);
             int netnum = getUint8(PACKET_SEQ_OFFSET);
-            if (!fetch0(NET_HEADER_SIZE, netlen))
+            if (!fetch0(NET_HEADER_SIZE, netlen)) {
-            {
+                logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
-                logger.warn("Reached end of input stream: packet #" + netnum
-                        + ", len = " + netlen);
                 return false;
             }
 
             // Detecting error code.
             final int mark = getUint8(NET_HEADER_SIZE);
-            if (mark != 0)
+            if (mark != 0) {
-            {
                 if (mark == 255) // error from master
                 {
-                    // Indicates an error, for example trying to fetch from wrong
+                    // Indicates an error, for example trying to fetch from
+                    // wrong
                     // binlog position.
                     position = NET_HEADER_SIZE + 1;
                     final int errno = getInt16();
                     String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
                     String errmsg = getFixString(limit - position);
-                    throw new IOException("Received error packet:"
+                    throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
-                            + " errno = " + errno + ", sqlstate = " + sqlstate
+                                          + " errmsg = " + errmsg);
-                            + " errmsg = " + errmsg);
+                } else if (mark == 254) {
-                }
-                else if (mark == 254)
-                {
                     // Indicates end of stream. It's not clear when this would
                     // be sent.
-                    logger.warn("Received EOF packet from server, apparent"
+                    logger.warn("Received EOF packet from server, apparent" + " master disconnected.");
-                            + " master disconnected.");
                     return false;
-                }
+                } else {
-                else
-                {
                     // Should not happen.
-                    throw new IOException("Unexpected response " + mark
+                    throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
-                            + " while fetching binlog: packet #" + netnum
+                                          + ", len = " + netlen);
-                            + ", len = " + netlen);
                 }
             }
 
             // The first packet is a multi-packet, concatenate the packets.
-            while (netlen == MAX_PACKET_LENGTH)
+            while (netlen == MAX_PACKET_LENGTH) {
-            {
+                if (!fetch0(0, NET_HEADER_SIZE)) {
-                if (!fetch0(0, NET_HEADER_SIZE))
-                {
                     logger.warn("Reached end of input stream while fetching header");
                     return false;
                 }
 
                 netlen = getUint24(PACKET_LEN_OFFSET);
                 netnum = getUint8(PACKET_SEQ_OFFSET);
-                if (!fetch0(limit, netlen))
+                if (!fetch0(limit, netlen)) {
-                {
+                    logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
-                    logger.warn("Reached end of input stream: packet #"
-                            + netnum + ", len = " + netlen);
                     return false;
                 }
             }
@@ -423,43 +338,32 @@ public final class DirectLogFetcher extends LogFetcher
             position = origin;
             limit -= origin;
             return true;
-        }
+        } catch (SocketTimeoutException e) {
-        catch (SocketTimeoutException e)
-        {
             close(); /* Do cleanup */
             logger.error("Socket timeout expired, closing connection", e);
             throw e;
-        }
+        } catch (InterruptedIOException e) {
-        catch (InterruptedIOException e)
-        {
             close(); /* Do cleanup */
             logger.warn("I/O interrupted while reading from client socket", e);
             throw e;
-        }
+        } catch (IOException e) {
-        catch (IOException e)
-        {
             close(); /* Do cleanup */
             logger.error("I/O error while reading from client socket", e);
             throw e;
         }
     }
 
-    private final boolean fetch0(final int off, final int len)
+    private final boolean fetch0(final int off, final int len) throws IOException {
-            throws IOException
-    {
         ensureCapacity(off + len);
 
-        for (int count, n = 0; n < len; n += count)
+        for (int count, n = 0; n < len; n += count) {
-        {
+            if (0 > (count = mysqlInput.read(buffer, off + n, len - n))) {
-            if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
-            {
                 // Reached end of input stream
                 return false;
             }
         }
 
-        if (limit < off + len)
+        if (limit < off + len) limit = off + len;
-            limit = off + len;
         return true;
     }
 
@@ -468,19 +372,14 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
      */
-    public void close() throws IOException
+    public void close() throws IOException {
-    {
+        try {
-        try
+            if (conn != null) conn.close();
-        {
-            if (conn != null)
-                conn.close();
 
             conn = null;
             mysqlInput = null;
             mysqlOutput = null;
-        }
+        } catch (SQLException e) {
-        catch (SQLException e)
-        {
             logger.warn("Unable to close connection", e);
         }
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java

@@ -129,11 +129,11 @@ public final class FileLogFetcher extends LogFetcher {
             }
         } else if (limit > 0) {
             System.arraycopy(buffer, origin, buffer, 0, limit);
+            position -= origin;
+            origin = 0;
             final int len = fin.read(buffer, limit, buffer.length - limit);
             if (len >= 0) {
                 limit += len;
-                position -= origin;
-                origin = 0;
 
                 /* More binlog to fetch */
                 return true;

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java

@@ -0,0 +1,418 @@
+package com.taobao.tddl.dbsync.binlog;
+
+/**
+ * 处理下MySQL json二进制转化为可读的字符串
+ * 
+ * @author agapple 2016年6月30日 上午11:26:17
+ * @since 1.0.22
+ */
+public class JsonConversion {
+
+    // JSON TYPE
+    public static final int  JSONB_TYPE_SMALL_OBJECT = 0x0;
+    public static final int  JSONB_TYPE_LARGE_OBJECT = 0x1;
+    public static final int  JSONB_TYPE_SMALL_ARRAY  = 0x2;
+    public static final int  JSONB_TYPE_LARGE_ARRAY  = 0x3;
+    public static final int  JSONB_TYPE_LITERAL      = 0x4;
+    public static final int  JSONB_TYPE_INT16        = 0x5;
+    public static final int  JSONB_TYPE_UINT16       = 0x6;
+    public static final int  JSONB_TYPE_INT32        = 0x7;
+    public static final int  JSONB_TYPE_UINT32       = 0x8;
+    public static final int  JSONB_TYPE_INT64        = 0x9;
+    public static final int  JSONB_TYPE_UINT64       = 0xA;
+    public static final int  JSONB_TYPE_DOUBLE       = 0xB;
+    public static final int  JSONB_TYPE_STRING       = 0xC;
+    public static final int  JSONB_TYPE_OPAQUE       = 0xF;
+    public static final char JSONB_NULL_LITERAL      = '\0';
+    public static final char JSONB_TRUE_LITERAL      = '\1';
+    public static final char JSONB_FALSE_LITERAL     = '\2';
+
+    /*
+     * The size of offset or size fields in the small and the large storage
+     * format for JSON objects and JSON arrays.
+     */
+    public static final int  SMALL_OFFSET_SIZE       = 2;
+    public static final int  LARGE_OFFSET_SIZE       = 4;
+
+    /*
+     * The size of key entries for objects when using the small storage format
+     * or the large storage format. In the small format it is 4 bytes (2 bytes
+     * for key length and 2 bytes for key offset). In the large format it is 6
+     * (2 bytes for length, 4 bytes for offset).
+     */
+    public static final int  KEY_ENTRY_SIZE_SMALL    = (2 + SMALL_OFFSET_SIZE);
+    public static final int  KEY_ENTRY_SIZE_LARGE    = (2 + LARGE_OFFSET_SIZE);
+
+    /*
+     * The size of value entries for objects or arrays. When using the small
+     * storage format, the entry size is 3 (1 byte for type, 2 bytes for
+     * offset). When using the large storage format, it is 5 (1 byte for type, 4
+     * bytes for offset).
+     */
+    public static final int  VALUE_ENTRY_SIZE_SMALL  = (1 + SMALL_OFFSET_SIZE);
+    public static final int  VALUE_ENTRY_SIZE_LARGE  = (1 + LARGE_OFFSET_SIZE);
+
+    public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
+        buffer = buffer.duplicate(buffer.position(), (int) len);
+        switch (type) {
+            case JSONB_TYPE_SMALL_OBJECT:
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
+            case JSONB_TYPE_LARGE_OBJECT:
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
+            case JSONB_TYPE_SMALL_ARRAY:
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
+            case JSONB_TYPE_LARGE_ARRAY:
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
+            default:
+                return parse_scalar(type, buffer, len, charsetName);
+        }
+    }
+
+    private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
+                                                    String charsetName) {
+        long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+        if (len < 2 * offset_size) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        long element_count = read_offset_or_size(buffer, large);
+        long bytes = read_offset_or_size(buffer, large);
+
+        if (bytes > len) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        long header_size = 2 * offset_size;
+        if (type == Json_enum_type.OBJECT) {
+            header_size += element_count * (large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL);
+        }
+
+        header_size += element_count * (large ? VALUE_ENTRY_SIZE_LARGE : VALUE_ENTRY_SIZE_SMALL);
+        if (header_size > bytes) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        return new Json_Value(type, buffer.rewind(), element_count, bytes, large);
+    }
+
+    private static long read_offset_or_size(LogBuffer buffer, boolean large) {
+        return large ? buffer.getUint32() : buffer.getUint16();
+    }
+
+    private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
+        switch (type) {
+            case JSONB_TYPE_LITERAL:
+                /* purecov: inspected */
+                int data = buffer.getUint8();
+                switch (data) {
+                    case JSONB_NULL_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_NULL);
+                    case JSONB_TRUE_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_TRUE);
+                    case JSONB_FALSE_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_FALSE);
+                    default:
+                        throw new IllegalArgumentException("illegal json data");
+                }
+            case JSONB_TYPE_INT16:
+                return new Json_Value(Json_enum_type.INT, buffer.getInt16());
+            case JSONB_TYPE_INT32:
+                return new Json_Value(Json_enum_type.INT, buffer.getInt32());
+            case JSONB_TYPE_INT64:
+                return new Json_Value(Json_enum_type.INT, buffer.getLong64());
+            case JSONB_TYPE_UINT16:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUint16());
+            case JSONB_TYPE_UINT32:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUint32());
+            case JSONB_TYPE_UINT64:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUlong64());
+            case JSONB_TYPE_DOUBLE:
+                return new Json_Value(Json_enum_type.DOUBLE, Double.valueOf(buffer.getDouble64()));
+            case JSONB_TYPE_STRING:
+                int max_bytes = (int) Math.min(len, 5);
+                long tlen = 0;
+                long str_len = 0;
+                long n = 0;
+                byte[] datas = buffer.getData(max_bytes);
+                for (int i = 0; i < max_bytes; i++) {
+                    // Get the next 7 bits of the length.
+                    tlen |= (datas[i] & 0x7f) << (7 * i);
+                    if ((datas[i] & 0x80) == 0) {
+                        // The length shouldn't exceed 32 bits.
+                        if (tlen > 4294967296L) {
+                            throw new IllegalArgumentException("illegal json data");
+                        }
+
+                        // This was the last byte. Return successfully.
+                        n = i + 1;
+                        str_len = tlen;
+                        break;
+                    }
+                }
+
+                if (len < n + str_len) {
+                    throw new IllegalArgumentException("illegal json data");
+                }
+                return new Json_Value(Json_enum_type.STRING, buffer.rewind()
+                    .forward((int) n)
+                    .getFixString((int) str_len, charsetName));
+            case JSONB_TYPE_OPAQUE:
+                /*
+                 * There should always be at least one byte, which tells the
+                 * field type of the opaque value.
+                 */
+                // The type is encoded as a uint8 that maps to an
+                // enum_field_types.
+                int type_byte = buffer.getUint8();
+                int position = buffer.position();
+                // Then there's the length of the value.
+                int q_max_bytes = (int) Math.min(len, 5);
+                long q_tlen = 0;
+                long q_str_len = 0;
+                long q_n = 0;
+                byte[] q_datas = buffer.getData(q_max_bytes);
+                for (int i = 0; i < q_max_bytes; i++) {
+                    // Get the next 7 bits of the length.
+                    q_tlen |= (q_datas[i] & 0x7f) << (7 * i);
+                    if ((q_datas[i] & 0x80) == 0) {
+                        // The length shouldn't exceed 32 bits.
+                        if (q_tlen > 4294967296L) {
+                            throw new IllegalArgumentException("illegal json data");
+                        }
+
+                        // This was the last byte. Return successfully.
+                        q_n = i + 1;
+                        q_str_len = q_tlen;
+                        break;
+                    }
+                }
+
+                if (q_str_len == 0 || len < q_n + q_str_len) {
+                    throw new IllegalArgumentException("illegal json data");
+                }
+                return new Json_Value(type_byte, buffer.position(position).forward((int) q_n), q_str_len);
+            default:
+                throw new IllegalArgumentException("illegal json data");
+        }
+    }
+
+    public static class Json_Value {
+
+        Json_enum_type m_type;
+        int            m_field_type;
+        LogBuffer      m_data;
+        long           m_element_count;
+        long           m_length;
+        String         m_string_value;
+        Number         m_int_value;
+        double         m_double_value;
+        boolean        m_large;
+
+        public Json_Value(Json_enum_type t){
+            this.m_type = t;
+        }
+
+        public Json_Value(Json_enum_type t, Number val){
+            this.m_type = t;
+            if (t == Json_enum_type.DOUBLE) {
+                this.m_double_value = val.doubleValue();
+            } else {
+                this.m_int_value = val;
+            }
+        }
+
+        public Json_Value(Json_enum_type t, String value){
+            this.m_type = t;
+            this.m_string_value = value;
+        }
+
+        public Json_Value(int field_type, LogBuffer data, long bytes){
+            this.m_type = Json_enum_type.OPAQUE; // 不确定类型
+            this.m_field_type = field_type;
+            this.m_data = data;
+            this.m_length = bytes;
+        }
+
+        public Json_Value(Json_enum_type t, LogBuffer data, long element_count, long bytes, boolean large){
+            this.m_type = t;
+            this.m_data = data;
+            this.m_element_count = element_count;
+            this.m_length = bytes;
+            this.m_large = large;
+        }
+
+        public String key(int i, String charsetName) {
+            m_data.rewind();
+            int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+            int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
+            int entry_offset = 2 * offset_size + key_entry_size * i;
+            // The offset of the key is the first part of the key
+            // entry.
+            m_data.forward(entry_offset);
+            long key_offset = read_offset_or_size(m_data, m_large);
+            // The length of the key is the second part of the
+            // entry, always two
+            // bytes.
+            long key_length = m_data.getUint16();
+            return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
+        }
+
+        public Json_Value element(int i, String charsetName) {
+            m_data.rewind();
+            int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+            int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
+            int value_entry_size = m_large ? VALUE_ENTRY_SIZE_LARGE : VALUE_ENTRY_SIZE_SMALL;
+            int first_entry_offset = 2 * offset_size;
+            if (m_type == Json_enum_type.OBJECT) {
+                first_entry_offset += m_element_count * key_entry_size;
+            }
+            int entry_offset = first_entry_offset + value_entry_size * i;
+            int type = m_data.forward(entry_offset).getUint8();
+            if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
+                || (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
+                return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
+            }
+            int value_offset = (int) read_offset_or_size(m_data, m_large);
+            return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
+        }
+
+        public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
+            switch (m_type) {
+                case OBJECT:
+                    buf.append("{");
+                    for (int i = 0; i < m_element_count; ++i) {
+                        if (i > 0) {
+                            buf.append(", ");
+                        }
+                        buf.append('"').append(key(i, charsetName)).append('"');
+                        buf.append(": ");
+                        element(i, charsetName).toJsonString(buf, charsetName);
+                    }
+                    buf.append("}");
+                    break;
+                case ARRAY:
+                    buf.append("[");
+                    for (int i = 0; i < m_element_count; ++i) {
+                        if (i > 0) {
+                            buf.append(", ");
+                        }
+                        element(i, charsetName).toJsonString(buf, charsetName);
+                    }
+                    buf.append("]");
+                    break;
+                case DOUBLE:
+                    buf.append(Double.valueOf(m_double_value).toString());
+                    break;
+                case INT:
+                    buf.append(m_int_value.toString());
+                    break;
+                case UINT:
+                    buf.append(m_int_value.toString());
+                    break;
+                case LITERAL_FALSE:
+                    buf.append("false");
+                    break;
+                case LITERAL_TRUE:
+                    buf.append("true");
+                    break;
+                case LITERAL_NULL:
+                    buf.append("NULL");
+                    break;
+                case OPAQUE:
+                    String text = null;
+                    if (m_field_type == LogEvent.MYSQL_TYPE_NEWDECIMAL) {
+                        int precision = m_data.getInt8();
+                        int scale = m_data.getInt8();
+                        text = m_data.getDecimal(precision, scale).toPlainString();
+                        buf.append(text);
+                    } else if (m_field_type == LogEvent.MYSQL_TYPE_TIME) {
+                        long packed_value = m_data.getLong64();
+                        if (packed_value == 0) {
+                            text = "00:00:00";
+                        } else {
+                            long ultime = Math.abs(packed_value);
+                            long intpart = ultime >> 24;
+                            int frac = (int) (ultime % (1L << 24));
+                            text = String.format("%s%02d:%02d:%02d",
+                                packed_value >= 0 ? "" : "-",
+                                (int) ((intpart >> 12) % (1 << 10)),
+                                (int) ((intpart >> 6) % (1 << 6)),
+                                (int) (intpart % (1 << 6)));
+                            text = text + "." + usecondsToStr(frac, 6);
+                        }
+                        buf.append('"').append(text).append('"');
+                    } else if (m_field_type == LogEvent.MYSQL_TYPE_DATE || m_field_type == LogEvent.MYSQL_TYPE_DATETIME
+                               || m_field_type == LogEvent.MYSQL_TYPE_TIMESTAMP) {
+                        long packed_value = m_data.getLong64();
+                        if (packed_value == 0) {
+                            text = "0000-00-00 00:00:00";
+                        } else {
+                            // 构造TimeStamp只处理到秒
+                            long ultime = Math.abs(packed_value);
+                            long intpart = ultime >> 24;
+                            int frac = (int) (ultime % (1L << 24));
+                            long ymd = intpart >> 17;
+                            long ym = ymd >> 5;
+                            long hms = intpart % (1 << 17);
+                            text = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                                (int) (ym / 13),
+                                (int) (ym % 13),
+                                (int) (ymd % (1 << 5)),
+                                (int) (hms >> 12),
+                                (int) ((hms >> 6) % (1 << 6)),
+                                (int) (hms % (1 << 6)));
+                            text = text + "." + usecondsToStr(frac, 6);
+                        }
+                        buf.append('"').append(text).append('"');
+                    } else {
+                        text = m_data.getFixString((int) m_length, charsetName);
+                        buf.append('"').append(escapse(text)).append('"');
+                    }
+
+                    break;
+                case STRING:
+                    buf.append('"').append(escapse(m_string_value)).append('"');
+                    break;
+                case ERROR:
+                    throw new IllegalArgumentException("illegal json data");
+            }
+
+            return buf;
+        }
+    }
+
+    private static StringBuilder escapse(String data) {
+        StringBuilder sb = new StringBuilder(data.length());
+        int endIndex = data.length();
+        for (int i = 0; i < endIndex; ++i) {
+            char c = data.charAt(i);
+            if (c == '"') {
+                sb.append('\\');
+            } else if (c == '\\') {
+                sb.append("\\");
+            }
+            sb.append(c);
+        }
+        return sb;
+    }
+
+    public static enum Json_enum_type {
+        OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR
+    }
+
+    private static String usecondsToStr(int frac, int meta) {
+        String sec = String.valueOf(frac);
+        if (meta > 6) {
+            throw new IllegalArgumentException("unknow useconds meta : " + meta);
+        }
+
+        if (sec.length() < 6) {
+            StringBuilder result = new StringBuilder(6);
+            int len = 6 - sec.length();
+            for (; len > 0; len--) {
+                result.append('0');
+            }
+            result.append(sec);
+            sec = result.toString();
+        }
+
+        return sec.substring(0, meta);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java

@@ -7,69 +7,56 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
 
 /**
- * TODO: Document Me!!
+ * TODO: Document Me!! NOTE: Log context will NOT write multi-threaded.
- * 
- * NOTE: Log context will NOT write multi-threaded.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogContext
+public final class LogContext {
-{
+
     private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
 
     private FormatDescriptionLogEvent         formatDescription;
 
     private LogPosition                       logPosition;
 
-    public LogContext()
+    public LogContext(){
-    {
         this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
     }
 
-    public LogContext(FormatDescriptionLogEvent descriptionEvent)
+    public LogContext(FormatDescriptionLogEvent descriptionEvent){
-    {
         this.formatDescription = descriptionEvent;
     }
 
-    public final LogPosition getLogPosition()
+    public final LogPosition getLogPosition() {
-    {
         return logPosition;
     }
 
-    public final void setLogPosition(LogPosition logPosition)
+    public final void setLogPosition(LogPosition logPosition) {
-    {
         this.logPosition = logPosition;
     }
 
-    public final FormatDescriptionLogEvent getFormatDescription()
+    public final FormatDescriptionLogEvent getFormatDescription() {
-    {
         return formatDescription;
     }
 
-    public final void setFormatDescription(
+    public final void setFormatDescription(FormatDescriptionLogEvent formatDescription) {
-            FormatDescriptionLogEvent formatDescription)
-    {
         this.formatDescription = formatDescription;
     }
 
-    public final void putTable(TableMapLogEvent mapEvent)
+    public final void putTable(TableMapLogEvent mapEvent) {
-    {
         mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
     }
 
-    public final TableMapLogEvent getTable(final long tableId)
+    public final TableMapLogEvent getTable(final long tableId) {
-    {
         return mapOfTable.get(Long.valueOf(tableId));
     }
 
-    public final void clearAllTables()
+    public final void clearAllTables() {
-    {
         mapOfTable.clear();
     }
 
-    public void reset()
+    public void reset() {
-    {
         formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
 
         mapOfTable.clear();

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -61,28 +61,24 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogDecoder
+public final class LogDecoder {
-{
+
     protected static final Log logger    = LogFactory.getLog(LogDecoder.class);
 
     protected final BitSet     handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
 
-    public LogDecoder()
+    public LogDecoder(){
-    {
     }
 
-    public LogDecoder(final int fromIndex, final int toIndex)
+    public LogDecoder(final int fromIndex, final int toIndex){
-    {
         handleSet.set(fromIndex, toIndex);
     }
 
-    public final void handle(final int fromIndex, final int toIndex)
+    public final void handle(final int fromIndex, final int toIndex) {
-    {
         handleSet.set(fromIndex, toIndex);
     }
 
-    public final void handle(final int flagIndex)
+    public final void handle(final int flagIndex) {
-    {
         handleSet.set(flagIndex);
     }
 
@@ -90,48 +86,32 @@ public final class LogDecoder
      * Decoding an event from binary-log buffer.
      * 
      * @return <code>UknownLogEvent</code> if event type is unknown or skipped,
-     *         <code>null</code> if buffer is not including a full event.
+     * <code>null</code> if buffer is not including a full event.
      */
-    public LogEvent decode(LogBuffer buffer, LogContext context)
+    public LogEvent decode(LogBuffer buffer, LogContext context) throws IOException {
-            throws IOException
-    {
         final int limit = buffer.limit();
 
-        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
+        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
-        {
+            LogHeader header = new LogHeader(buffer, context.getFormatDescription());
-            LogHeader header = new LogHeader(buffer,
-                    context.getFormatDescription());
 
             final int len = header.getEventLen();
-            if (limit >= len)
+            if (limit >= len) {
-            {
                 LogEvent event;
 
                 /* Checking binary-log's header */
-                if (handleSet.get(header.getType()))
+                if (handleSet.get(header.getType())) {
-                {
                     buffer.limit(len);
-                    try
+                    try {
-                    {
                         /* Decoding binary-log to event */
                         event = decode(buffer, header, context);
-                    }
+                    } catch (IOException e) {
-                    catch (IOException e)
+                        if (logger.isWarnEnabled()) logger.warn("Decoding " + LogEvent.getTypeName(header.getType())
-                    {
+                                                                + " failed from: " + context.getLogPosition(), e);
-                        if (logger.isWarnEnabled())
-                            logger.warn("Decoding "
-                                    + LogEvent.getTypeName(header.getType())
-                                    + " failed from: "
-                                    + context.getLogPosition(), e);
                         throw e;
-                    }
+                    } finally {
-                    finally
-                    {
                         buffer.limit(limit); /* Restore limit */
                     }
-                }
+                } else {
-                else
-                {
                     /* Ignore unsupported binary-log. */
                     event = new UnknownLogEvent(header);
                 }
@@ -152,16 +132,14 @@ public final class LogDecoder
      * 
      * @return <code>UknownLogEvent</code> if event type is unknown or skipped.
      */
-    public static LogEvent decode(LogBuffer buffer, LogHeader header,
+    public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext context) throws IOException {
-            LogContext context) throws IOException
-    {
         FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
         LogPosition logPosition = context.getLogPosition();
 
         int checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
         if (header.getType() != LogEvent.FORMAT_DESCRIPTION_EVENT) {
             checksumAlg = descriptionEvent.header.getChecksumAlg();
-        }else {
+        } else {
             // 如果是format事件自己，也需要处理checksum
             checksumAlg = header.getChecksumAlg();
         }
@@ -171,319 +149,258 @@ public final class LogDecoder
             buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
         }
 
-        switch (header.getType())
+        switch (header.getType()) {
-        {
+            case LogEvent.QUERY_EVENT: {
-        case LogEvent.QUERY_EVENT:
+                QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
-            {
-                QueryLogEvent event = new QueryLogEvent(header, buffer,
-                        descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.XID_EVENT:
+            case LogEvent.XID_EVENT: {
-            {
+                XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
-                XidLogEvent event = new XidLogEvent(header, buffer,
-                        descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.TABLE_MAP_EVENT:
+            case LogEvent.TABLE_MAP_EVENT: {
-            {
+                TableMapLogEvent mapEvent = new TableMapLogEvent(header, buffer, descriptionEvent);
-                TableMapLogEvent mapEvent = new TableMapLogEvent(header,
-                        buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 context.putTable(mapEvent);
                 return mapEvent;
             }
-        case LogEvent.WRITE_ROWS_EVENT_V1:
+            case LogEvent.WRITE_ROWS_EVENT_V1: {
-            {
-                RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                event.fillTable(context);
-                return event;
-            }
-        case LogEvent.UPDATE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                event.fillTable(context);
-                return event;
-            }
-        case LogEvent.DELETE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                event.fillTable(context);
-                return event;
-            }
-        case LogEvent.ROTATE_EVENT:
-            {
-                RotateLogEvent event = new RotateLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition = new LogPosition(event.getFilename(),
-                        event.getPosition());
-                context.setLogPosition(logPosition);
-                return event;
-            }
-        case LogEvent.LOAD_EVENT:
-        case LogEvent.NEW_LOAD_EVENT:
-            {
-                LoadLogEvent event = new LoadLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported SLAVE_EVENT from: "
-                            + context.getLogPosition());
-                break;
-            }
-        case LogEvent.CREATE_FILE_EVENT:
-            {
-                CreateFileLogEvent event = new CreateFileLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.APPEND_BLOCK_EVENT:
-            {
-                AppendBlockLogEvent event = new AppendBlockLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.DELETE_FILE_EVENT:
-            {
-                DeleteFileLogEvent event = new DeleteFileLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.EXEC_LOAD_EVENT:
-            {
-                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.START_EVENT_V3:
-            {
-                /* This is sent only by MySQL <=4.x */
-                StartLogEventV3 event = new StartLogEventV3(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.STOP_EVENT:
-            {
-                StopLogEvent event = new StopLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.INTVAR_EVENT:
-            {
-                IntvarLogEvent event = new IntvarLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.RAND_EVENT:
-            {
-                RandLogEvent event = new RandLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.USER_VAR_EVENT:
-            {
-                UserVarLogEvent event = new UserVarLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.FORMAT_DESCRIPTION_EVENT:
-            {
-                descriptionEvent = new FormatDescriptionLogEvent(header,
-                        buffer, descriptionEvent);
-                context.setFormatDescription(descriptionEvent);
-                return descriptionEvent;
-            }
-        case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
-                            + context.getLogPosition());
-                // ev = new Write_rows_log_event_old(buf, event_len,
-                // description_event);
-                break;
-            }
-        case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
-                            + context.getLogPosition());
-                // ev = new Update_rows_log_event_old(buf, event_len,
-                // description_event);
-                break;
-            }
-        case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
-                            + context.getLogPosition());
-                // ev = new Delete_rows_log_event_old(buf, event_len,
-                // description_event);
-                break;
-            }
-        case LogEvent.BEGIN_LOAD_QUERY_EVENT:
-            {
-                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
-                        header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
-            {
-                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
-                        header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.INCIDENT_EVENT:
-            {
-                IncidentLogEvent event = new IncidentLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.HEARTBEAT_LOG_EVENT:
-        {
-            HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
-                    descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-        }
-        case LogEvent.IGNORABLE_LOG_EVENT:
-            {
-                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.ROWS_QUERY_LOG_EVENT:
-            {
-                RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.WRITE_ROWS_EVENT: {
                 RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.UPDATE_ROWS_EVENT: {
+            case LogEvent.UPDATE_ROWS_EVENT_V1: {
                 RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.DELETE_ROWS_EVENT: {
+            case LogEvent.DELETE_ROWS_EVENT_V1: {
                 RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.GTID_LOG_EVENT:
+            case LogEvent.ROTATE_EVENT: {
-        case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
+                RotateLogEvent event = new RotateLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition = new LogPosition(event.getFilename(), event.getPosition());
+                context.setLogPosition(logPosition);
+                return event;
+            }
+            case LogEvent.LOAD_EVENT:
+            case LogEvent.NEW_LOAD_EVENT: {
+                LoadLogEvent event = new LoadLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
             {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: "
+                                                        + context.getLogPosition());
+                break;
+            }
+            case LogEvent.CREATE_FILE_EVENT: {
+                CreateFileLogEvent event = new CreateFileLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.APPEND_BLOCK_EVENT: {
+                AppendBlockLogEvent event = new AppendBlockLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.DELETE_FILE_EVENT: {
+                DeleteFileLogEvent event = new DeleteFileLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.EXEC_LOAD_EVENT: {
+                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.START_EVENT_V3: {
+                /* This is sent only by MySQL <=4.x */
+                StartLogEventV3 event = new StartLogEventV3(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.STOP_EVENT: {
+                StopLogEvent event = new StopLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.INTVAR_EVENT: {
+                IntvarLogEvent event = new IntvarLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.RAND_EVENT: {
+                RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.USER_VAR_EVENT: {
+                UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.FORMAT_DESCRIPTION_EVENT: {
+                descriptionEvent = new FormatDescriptionLogEvent(header, buffer, descriptionEvent);
+                context.setFormatDescription(descriptionEvent);
+                return descriptionEvent;
+            }
+            case LogEvent.PRE_GA_WRITE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+                                                        + context.getLogPosition());
+                // ev = new Write_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+            case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+                                                        + context.getLogPosition());
+                // ev = new Update_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+            case LogEvent.PRE_GA_DELETE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+                                                        + context.getLogPosition());
+                // ev = new Delete_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+            case LogEvent.BEGIN_LOAD_QUERY_EVENT: {
+                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.EXECUTE_LOAD_QUERY_EVENT: {
+                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.INCIDENT_EVENT: {
+                IncidentLogEvent event = new IncidentLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.HEARTBEAT_LOG_EVENT: {
+                HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.IGNORABLE_LOG_EVENT: {
+                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.ROWS_QUERY_LOG_EVENT: {
+                RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.WRITE_ROWS_EVENT: {
+                RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+            case LogEvent.UPDATE_ROWS_EVENT: {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+            case LogEvent.DELETE_ROWS_EVENT: {
+                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+            case LogEvent.GTID_LOG_EVENT:
+            case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
                 GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
+            case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
-            {
                 PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.ANNOTATE_ROWS_EVENT:
+            case LogEvent.ANNOTATE_ROWS_EVENT: {
-            {
+                AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
-            AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        case LogEvent.BINLOG_CHECKPOINT_EVENT:
-            {
-            BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        case LogEvent.GTID_EVENT:
-            {
-            MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        case LogEvent.GTID_LIST_EVENT:
-            {
-            MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        default:
-            /*
-                Create an object of Ignorable_log_event for unrecognized sub-class.
-                So that SLAVE SQL THREAD will only update the position and continue.
-             */
-            if((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0){
-                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
-            }else {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unrecognized binlog event "
-                            + LogEvent.getTypeName(header.getType()) + " from: "
-                            + context.getLogPosition());
             }
+            case LogEvent.BINLOG_CHECKPOINT_EVENT: {
+                BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.GTID_EVENT: {
+                MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.GTID_LIST_EVENT: {
+                MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            default:
+                /*
+                 * Create an object of Ignorable_log_event for unrecognized
+                 * sub-class. So that SLAVE SQL THREAD will only update the
+                 * position and continue.
+                 */
+                if ((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0) {
+                    IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
+                    /* updating position in context */
+                    logPosition.position = header.getLogPos();
+                    return event;
+                } else {
+                    if (logger.isWarnEnabled()) logger.warn("Skipping unrecognized binlog event "
+                                                            + LogEvent.getTypeName(header.getType()) + " from: "
+                                                            + context.getLogPosition());
+                }
         }
 
         /* updating position in context */

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java

@@ -7,238 +7,217 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
 
 /**
  * Binary log event definitions. This includes generic code common to all types
- * of log events, as well as specific code for each type of log event.
+ * of log events, as well as specific code for each type of log event. - All
- * 
+ * numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored in
- * - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
+ * little endian, i.e., the least significant byte first, unless otherwise
- * in little endian, i.e., the least significant byte first, unless otherwise
+ * specified. representation of unsigned integers, called Packed Integer. A
- * specified.
+ * Packed Integer has the capacity of storing up to 8-byte integers, while small
- * 
+ * integers still can use 1, 3, or 4 bytes. The value of the first byte
- * representation of unsigned integers, called Packed Integer. A Packed Integer
+ * determines how to read the number, according to the following table:
- * has the capacity of storing up to 8-byte integers, while small integers still
- * can use 1, 3, or 4 bytes. The value of the first byte determines how to read
- * the number, according to the following table:
- * 
  * <table>
  * <caption>Format of Packed Integer</caption>
- * 
  * <tr>
  * <th>First byte</th>
  * <th>Format</th>
  * </tr>
- * 
  * <tr>
  * <td>0-250</td>
  * <td>The first byte is the number (in the range 0-250), and no more bytes are
  * used.</td>
  * </tr>
- * 
  * <tr>
  * <td>252</td>
  * <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
  * </tr>
- * 
  * <tr>
  * <td>253</td>
  * <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
  * </tr>
- * 
  * <tr>
  * <td>254</td>
  * <td>Eight more bytes are used. The number is in the range
  * 0xffffff-0xffffffffffffffff.</td>
  * </tr>
- * 
  * </table>
- * 
  * - Strings are stored in various formats. The format of each string is
  * documented separately.
  * 
  * @see mysql-5.1.60/sql/log_event.h
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class LogEvent
+public abstract class LogEvent {
-{
+
     /*
-     * 3 is MySQL 4.x; 4 is MySQL 5.0.0.
+     * 3 is MySQL 4.x; 4 is MySQL 5.0.0. Compared to version 3, version 4 has: -
-     * Compared to version 3, version 4 has:
+     * a different Start_log_event, which includes info about the binary log
-     * - a different Start_log_event, which includes info about the binary log
      * (sizes of headers); this info is included for better compatibility if the
-     * master's MySQL version is different from the slave's.
+     * master's MySQL version is different from the slave's. - all events have a
-     * - all events have a unique ID (the triplet (server_id, timestamp at server
+     * unique ID (the triplet (server_id, timestamp at server start, other) to
-     * start, other) to be sure an event is not executed more than once in a
+     * be sure an event is not executed more than once in a multimaster setup,
-     * multimaster setup, example:
+     * example: M1 / \ v v M2 M3 \ / v v S if a query is run on M1, it will
-     *              M1
+     * arrive twice on S, so we need that S remembers the last unique ID it has
-     *            /   \
+     * processed, to compare and know if the event should be skipped or not.
-     *           v     v
+     * Example of ID: we already have the server id (4 bytes), plus:
-     *           M2    M3
-     *           \     /
-     *            v   v
-     *              S
-     * if a query is run on M1, it will arrive twice on S, so we need that S
-     * remembers the last unique ID it has processed, to compare and know if the
-     * event should be skipped or not. Example of ID: we already have the server id
-     * (4 bytes), plus:
      * timestamp_when_the_master_started (4 bytes), a counter (a sequence number
      * which increments every time we write an event to the binlog) (3 bytes).
      * Q: how do we handle when the counter is overflowed and restarts from 0 ?
-     * 
      * - Query and Load (Create or Execute) events may have a more precise
-     *   timestamp (with microseconds), number of matched/affected/warnings rows
+     * timestamp (with microseconds), number of matched/affected/warnings rows
-     * and fields of session variables: SQL_MODE,
+     * and fields of session variables: SQL_MODE, FOREIGN_KEY_CHECKS,
-     * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
+     * UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and charsets, the
-     * charsets, the PASSWORD() version (old/new/...).
+     * PASSWORD() version (old/new/...).
      */
-    public static final int    BINLOG_VERSION           = 4;
+    public static final int    BINLOG_VERSION                           = 4;
 
     /* Default 5.0 server version */
-    public static final String SERVER_VERSION           = "5.0";
+    public static final String SERVER_VERSION                           = "5.0";
 
     /**
      * Event header offsets; these point to places inside the fixed header.
      */
-    public static final int    EVENT_TYPE_OFFSET        = 4;
+    public static final int    EVENT_TYPE_OFFSET                        = 4;
-    public static final int    SERVER_ID_OFFSET         = 5;
+    public static final int    SERVER_ID_OFFSET                         = 5;
-    public static final int    EVENT_LEN_OFFSET         = 9;
+    public static final int    EVENT_LEN_OFFSET                         = 9;
-    public static final int    LOG_POS_OFFSET           = 13;
+    public static final int    LOG_POS_OFFSET                           = 13;
-    public static final int    FLAGS_OFFSET             = 17;
+    public static final int    FLAGS_OFFSET                             = 17;
 
     /* event-specific post-header sizes */
     // where 3.23, 4.x and 5.0 agree
-    public static final int    QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2);
+    public static final int    QUERY_HEADER_MINIMAL_LEN                 = (4 + 4 + 1 + 2);
     // where 5.0 differs: 2 for len of N-bytes vars.
-    public static final int    QUERY_HEADER_LEN         = (QUERY_HEADER_MINIMAL_LEN + 2);
+    public static final int    QUERY_HEADER_LEN                         = (QUERY_HEADER_MINIMAL_LEN + 2);
 
     /* Enumeration type for the different types of log events. */
-    public static final int    UNKNOWN_EVENT            = 0;
+    public static final int    UNKNOWN_EVENT                            = 0;
-    public static final int    START_EVENT_V3           = 1;
+    public static final int    START_EVENT_V3                           = 1;
-    public static final int    QUERY_EVENT              = 2;
+    public static final int    QUERY_EVENT                              = 2;
-    public static final int    STOP_EVENT               = 3;
+    public static final int    STOP_EVENT                               = 3;
-    public static final int    ROTATE_EVENT             = 4;
+    public static final int    ROTATE_EVENT                             = 4;
-    public static final int    INTVAR_EVENT             = 5;
+    public static final int    INTVAR_EVENT                             = 5;
-    public static final int    LOAD_EVENT               = 6;
+    public static final int    LOAD_EVENT                               = 6;
-    public static final int    SLAVE_EVENT              = 7;
+    public static final int    SLAVE_EVENT                              = 7;
-    public static final int    CREATE_FILE_EVENT        = 8;
+    public static final int    CREATE_FILE_EVENT                        = 8;
-    public static final int    APPEND_BLOCK_EVENT       = 9;
+    public static final int    APPEND_BLOCK_EVENT                       = 9;
-    public static final int    EXEC_LOAD_EVENT          = 10;
+    public static final int    EXEC_LOAD_EVENT                          = 10;
-    public static final int    DELETE_FILE_EVENT        = 11;
+    public static final int    DELETE_FILE_EVENT                        = 11;
 
     /**
      * NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer sql_ex,
      * allowing multibyte TERMINATED BY etc; both types share the same class
      * (Load_log_event)
      */
-    public static final int    NEW_LOAD_EVENT           = 12;
+    public static final int    NEW_LOAD_EVENT                           = 12;
-    public static final int    RAND_EVENT               = 13;
+    public static final int    RAND_EVENT                               = 13;
-    public static final int    USER_VAR_EVENT           = 14;
+    public static final int    USER_VAR_EVENT                           = 14;
-    public static final int    FORMAT_DESCRIPTION_EVENT = 15;
+    public static final int    FORMAT_DESCRIPTION_EVENT                 = 15;
-    public static final int    XID_EVENT                = 16;
+    public static final int    XID_EVENT                                = 16;
-    public static final int    BEGIN_LOAD_QUERY_EVENT   = 17;
+    public static final int    BEGIN_LOAD_QUERY_EVENT                   = 17;
-    public static final int    EXECUTE_LOAD_QUERY_EVENT = 18;
+    public static final int    EXECUTE_LOAD_QUERY_EVENT                 = 18;
 
-    public static final int    TABLE_MAP_EVENT          = 19;
+    public static final int    TABLE_MAP_EVENT                          = 19;
 
     /**
      * These event numbers were used for 5.1.0 to 5.1.15 and are therefore
      * obsolete.
      */
-    public static final int    PRE_GA_WRITE_ROWS_EVENT  = 20;
+    public static final int    PRE_GA_WRITE_ROWS_EVENT                  = 20;
-    public static final int    PRE_GA_UPDATE_ROWS_EVENT = 21;
+    public static final int    PRE_GA_UPDATE_ROWS_EVENT                 = 21;
-    public static final int    PRE_GA_DELETE_ROWS_EVENT = 22;
+    public static final int    PRE_GA_DELETE_ROWS_EVENT                 = 22;
 
     /**
      * These event numbers are used from 5.1.16 and forward
      */
-    public static final int    WRITE_ROWS_EVENT_V1      = 23;
+    public static final int    WRITE_ROWS_EVENT_V1                      = 23;
-    public static final int    UPDATE_ROWS_EVENT_V1     = 24;
+    public static final int    UPDATE_ROWS_EVENT_V1                     = 24;
-    public static final int    DELETE_ROWS_EVENT_V1     = 25;
+    public static final int    DELETE_ROWS_EVENT_V1                     = 25;
 
     /**
      * Something out of the ordinary happened on the master
      */
-    public static final int    INCIDENT_EVENT           = 26;
+    public static final int    INCIDENT_EVENT                           = 26;
 
     /**
-     * Heartbeat event to be send by master at its idle time to ensure master's online status to slave
+     * Heartbeat event to be send by master at its idle time to ensure master's
+     * online status to slave
      */
-    public static final int    HEARTBEAT_LOG_EVENT      = 27;
+    public static final int    HEARTBEAT_LOG_EVENT                      = 27;
 
     /**
-     * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
+     * In some situations, it is necessary to send over ignorable data to the
-     * case there is code for handling it, but which can be ignored if it is not recognized.
+     * slave: data that a slave can handle in case there is code for handling
+     * it, but which can be ignored if it is not recognized.
      */
-    public static final int    IGNORABLE_LOG_EVENT      = 28;
+    public static final int    IGNORABLE_LOG_EVENT                      = 28;
-    public static final int    ROWS_QUERY_LOG_EVENT     = 29;
+    public static final int    ROWS_QUERY_LOG_EVENT                     = 29;
 
     /** Version 2 of the Row events */
-    public static final int    WRITE_ROWS_EVENT         = 30;
+    public static final int    WRITE_ROWS_EVENT                         = 30;
-    public static final int    UPDATE_ROWS_EVENT        = 31;
+    public static final int    UPDATE_ROWS_EVENT                        = 31;
-    public static final int    DELETE_ROWS_EVENT        = 32;
+    public static final int    DELETE_ROWS_EVENT                        = 32;
 
-    public static final int    GTID_LOG_EVENT           = 33;
+    public static final int    GTID_LOG_EVENT                           = 33;
-    public static final int    ANONYMOUS_GTID_LOG_EVENT = 34;
+    public static final int    ANONYMOUS_GTID_LOG_EVENT                 = 34;
 
-    public static final int    PREVIOUS_GTIDS_LOG_EVENT = 35;
+    public static final int    PREVIOUS_GTIDS_LOG_EVENT                 = 35;
 
     // mariaDb 5.5.34
     /* New MySQL/Sun events are to be added right above this comment */
-    public static final int    MYSQL_EVENTS_END         = 36;
+    public static final int    MYSQL_EVENTS_END                         = 36;
 
-    public static final int    MARIA_EVENTS_BEGIN       = 160;
+    public static final int    MARIA_EVENTS_BEGIN                       = 160;
     /* New Maria event numbers start from here */
-    public static final int    ANNOTATE_ROWS_EVENT      = 160;
+    public static final int    ANNOTATE_ROWS_EVENT                      = 160;
     /*
-    Binlog checkpoint event. Used for XA crash recovery on the master, not used
+     * Binlog checkpoint event. Used for XA crash recovery on the master, not
-    in replication.
+     * used in replication. A binlog checkpoint event specifies a binlog file
-    A binlog checkpoint event specifies a binlog file such that XA crash
+     * such that XA crash recovery can start from that file - and it is
-    recovery can start from that file - and it is guaranteed to find all XIDs
+     * guaranteed to find all XIDs that are prepared in storage engines but not
-    that are prepared in storage engines but not yet committed.
+     * yet committed.
      */
-    public static final int     BINLOG_CHECKPOINT_EVENT = 161;
+    public static final int    BINLOG_CHECKPOINT_EVENT                  = 161;
     /*
-    Gtid event. For global transaction ID, used to start a new event group,
+     * Gtid event. For global transaction ID, used to start a new event group,
-    instead of the old BEGIN query event, and also to mark stand-alone
+     * instead of the old BEGIN query event, and also to mark stand-alone
-    events.
+     * events.
      */
-    public static final int    GTID_EVENT               = 162;
+    public static final int    GTID_EVENT                               = 162;
     /*
-    Gtid list event. Logged at the start of every binlog, to record the
+     * Gtid list event. Logged at the start of every binlog, to record the
-    current replication state. This consists of the last GTID seen for
+     * current replication state. This consists of the last GTID seen for each
-    each replication domain.
+     * replication domain.
      */
-    public static final int    GTID_LIST_EVENT          = 163;
+    public static final int    GTID_LIST_EVENT                          = 163;
 
     /** end marker */
-    public static final int    ENUM_END_EVENT           = 164;
+    public static final int    ENUM_END_EVENT                           = 164;
 
     /**
-    1 byte length, 1 byte format
+     * 1 byte length, 1 byte format Length is total length in bytes, including 2
-    Length is total length in bytes, including 2 byte header
+     * byte header Length values 0 and 1 are currently invalid and reserved.
-    Length values 0 and 1 are currently invalid and reserved.
+     */
-    */
+    public static final int    EXTRA_ROW_INFO_LEN_OFFSET                = 0;
-    public static final int    EXTRA_ROW_INFO_LEN_OFFSET    = 0;
+    public static final int    EXTRA_ROW_INFO_FORMAT_OFFSET             = 1;
-    public static final int    EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
+    public static final int    EXTRA_ROW_INFO_HDR_BYTES                 = 2;
-    public static final int    EXTRA_ROW_INFO_HDR_BYTES     = 2;
+    public static final int    EXTRA_ROW_INFO_MAX_PAYLOAD               = (255 - EXTRA_ROW_INFO_HDR_BYTES);
-    public static final int    EXTRA_ROW_INFO_MAX_PAYLOAD   = (255 - EXTRA_ROW_INFO_HDR_BYTES);
 
     // Events are without checksum though its generator
-    public static final int    BINLOG_CHECKSUM_ALG_OFF      = 0;
+    public static final int    BINLOG_CHECKSUM_ALG_OFF                  = 0;
     // is checksum-capable New Master (NM).
     // CRC32 of zlib algorithm.
-    public static final int    BINLOG_CHECKSUM_ALG_CRC32    = 1;
+    public static final int    BINLOG_CHECKSUM_ALG_CRC32                = 1;
     // the cut line: valid alg range is [1, 0x7f].
-    public static final int    BINLOG_CHECKSUM_ALG_ENUM_END = 2;
+    public static final int    BINLOG_CHECKSUM_ALG_ENUM_END             = 2;
     // special value to tag undetermined yet checksum
-    public static final int    BINLOG_CHECKSUM_ALG_UNDEF    = 255;
+    public static final int    BINLOG_CHECKSUM_ALG_UNDEF                = 255;
     // or events from checksum-unaware servers
 
-    public static final int    CHECKSUM_CRC32_SIGNATURE_LEN = 4;
+    public static final int    CHECKSUM_CRC32_SIGNATURE_LEN             = 4;
-    public static final int    BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
+    public static final int    BINLOG_CHECKSUM_ALG_DESC_LEN             = 1;
     /**
      * defined statically while there is just one alg implemented
      */
-    public static final int    BINLOG_CHECKSUM_LEN          = CHECKSUM_CRC32_SIGNATURE_LEN;
+    public static final int    BINLOG_CHECKSUM_LEN                      = CHECKSUM_CRC32_SIGNATURE_LEN;
 
     /* MySQL or old MariaDB slave with no announced capability. */
     public static final int    MARIA_SLAVE_CAPABILITY_UNKNOWN           = 0;
@@ -260,122 +239,120 @@ public abstract class LogEvent
     public static final int    MARIA_SLAVE_CAPABILITY_MINE              = MARIA_SLAVE_CAPABILITY_GTID;
 
     /**
-        For an event, 'e', carrying a type code, that a slave,
+     * For an event, 'e', carrying a type code, that a slave, 's', does not
-        's', does not recognize, 's' will check 'e' for
+     * recognize, 's' will check 'e' for LOG_EVENT_IGNORABLE_F, and if the flag
-        LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
+     * is set, then 'e' is ignored. Otherwise, 's' acknowledges that it has
-        is ignored. Otherwise, 's' acknowledges that it has
+     * found an unknown event in the relay log.
-        found an unknown event in the relay log.
      */
-    public static final int   LOG_EVENT_IGNORABLE_F    = 0x80;
+    public static final int    LOG_EVENT_IGNORABLE_F                    = 0x80;
 
     /** enum_field_types */
-    public static final int    MYSQL_TYPE_DECIMAL       = 0;
+    public static final int    MYSQL_TYPE_DECIMAL                       = 0;
-    public static final int    MYSQL_TYPE_TINY          = 1;
+    public static final int    MYSQL_TYPE_TINY                          = 1;
-    public static final int    MYSQL_TYPE_SHORT         = 2;
+    public static final int    MYSQL_TYPE_SHORT                         = 2;
-    public static final int    MYSQL_TYPE_LONG          = 3;
+    public static final int    MYSQL_TYPE_LONG                          = 3;
-    public static final int    MYSQL_TYPE_FLOAT         = 4;
+    public static final int    MYSQL_TYPE_FLOAT                         = 4;
-    public static final int    MYSQL_TYPE_DOUBLE        = 5;
+    public static final int    MYSQL_TYPE_DOUBLE                        = 5;
-    public static final int    MYSQL_TYPE_NULL          = 6;
+    public static final int    MYSQL_TYPE_NULL                          = 6;
-    public static final int    MYSQL_TYPE_TIMESTAMP     = 7;
+    public static final int    MYSQL_TYPE_TIMESTAMP                     = 7;
-    public static final int    MYSQL_TYPE_LONGLONG      = 8;
+    public static final int    MYSQL_TYPE_LONGLONG                      = 8;
-    public static final int    MYSQL_TYPE_INT24         = 9;
+    public static final int    MYSQL_TYPE_INT24                         = 9;
-    public static final int    MYSQL_TYPE_DATE          = 10;
+    public static final int    MYSQL_TYPE_DATE                          = 10;
-    public static final int    MYSQL_TYPE_TIME          = 11;
+    public static final int    MYSQL_TYPE_TIME                          = 11;
-    public static final int    MYSQL_TYPE_DATETIME      = 12;
+    public static final int    MYSQL_TYPE_DATETIME                      = 12;
-    public static final int    MYSQL_TYPE_YEAR          = 13;
+    public static final int    MYSQL_TYPE_YEAR                          = 13;
-    public static final int    MYSQL_TYPE_NEWDATE       = 14;
+    public static final int    MYSQL_TYPE_NEWDATE                       = 14;
-    public static final int    MYSQL_TYPE_VARCHAR       = 15;
+    public static final int    MYSQL_TYPE_VARCHAR                       = 15;
-    public static final int    MYSQL_TYPE_BIT           = 16;
+    public static final int    MYSQL_TYPE_BIT                           = 16;
-    public static final int    MYSQL_TYPE_TIMESTAMP2    = 17;
+    public static final int    MYSQL_TYPE_TIMESTAMP2                    = 17;
-    public static final int    MYSQL_TYPE_DATETIME2     = 18;
+    public static final int    MYSQL_TYPE_DATETIME2                     = 18;
-    public static final int    MYSQL_TYPE_TIME2         = 19;
+    public static final int    MYSQL_TYPE_TIME2                         = 19;
-    public static final int    MYSQL_TYPE_NEWDECIMAL    = 246;
+    public static final int    MYSQL_TYPE_JSON                          = 245;
-    public static final int    MYSQL_TYPE_ENUM          = 247;
+    public static final int    MYSQL_TYPE_NEWDECIMAL                    = 246;
-    public static final int    MYSQL_TYPE_SET           = 248;
+    public static final int    MYSQL_TYPE_ENUM                          = 247;
-    public static final int    MYSQL_TYPE_TINY_BLOB     = 249;
+    public static final int    MYSQL_TYPE_SET                           = 248;
-    public static final int    MYSQL_TYPE_MEDIUM_BLOB   = 250;
+    public static final int    MYSQL_TYPE_TINY_BLOB                     = 249;
-    public static final int    MYSQL_TYPE_LONG_BLOB     = 251;
+    public static final int    MYSQL_TYPE_MEDIUM_BLOB                   = 250;
-    public static final int    MYSQL_TYPE_BLOB          = 252;
+    public static final int    MYSQL_TYPE_LONG_BLOB                     = 251;
-    public static final int    MYSQL_TYPE_VAR_STRING    = 253;
+    public static final int    MYSQL_TYPE_BLOB                          = 252;
-    public static final int    MYSQL_TYPE_STRING        = 254;
+    public static final int    MYSQL_TYPE_VAR_STRING                    = 253;
-    public static final int    MYSQL_TYPE_GEOMETRY      = 255;
+    public static final int    MYSQL_TYPE_STRING                        = 254;
+    public static final int    MYSQL_TYPE_GEOMETRY                      = 255;
 
-    public static String getTypeName(final int type)
+    public static String getTypeName(final int type) {
-    {
+        switch (type) {
-        switch (type)
+            case START_EVENT_V3:
-        {
+                return "Start_v3";
-        case START_EVENT_V3:
+            case STOP_EVENT:
-            return "Start_v3";
+                return "Stop";
-        case STOP_EVENT:
+            case QUERY_EVENT:
-            return "Stop";
+                return "Query";
-        case QUERY_EVENT:
+            case ROTATE_EVENT:
-            return "Query";
+                return "Rotate";
-        case ROTATE_EVENT:
+            case INTVAR_EVENT:
-            return "Rotate";
+                return "Intvar";
-        case INTVAR_EVENT:
+            case LOAD_EVENT:
-            return "Intvar";
+                return "Load";
-        case LOAD_EVENT:
+            case NEW_LOAD_EVENT:
-            return "Load";
+                return "New_load";
-        case NEW_LOAD_EVENT:
+            case SLAVE_EVENT:
-            return "New_load";
+                return "Slave";
-        case SLAVE_EVENT:
+            case CREATE_FILE_EVENT:
-            return "Slave";
+                return "Create_file";
-        case CREATE_FILE_EVENT:
+            case APPEND_BLOCK_EVENT:
-            return "Create_file";
+                return "Append_block";
-        case APPEND_BLOCK_EVENT:
+            case DELETE_FILE_EVENT:
-            return "Append_block";
+                return "Delete_file";
-        case DELETE_FILE_EVENT:
+            case EXEC_LOAD_EVENT:
-            return "Delete_file";
+                return "Exec_load";
-        case EXEC_LOAD_EVENT:
+            case RAND_EVENT:
-            return "Exec_load";
+                return "RAND";
-        case RAND_EVENT:
+            case XID_EVENT:
-            return "RAND";
+                return "Xid";
-        case XID_EVENT:
+            case USER_VAR_EVENT:
-            return "Xid";
+                return "User var";
-        case USER_VAR_EVENT:
+            case FORMAT_DESCRIPTION_EVENT:
-            return "User var";
+                return "Format_desc";
-        case FORMAT_DESCRIPTION_EVENT:
+            case TABLE_MAP_EVENT:
-            return "Format_desc";
+                return "Table_map";
-        case TABLE_MAP_EVENT:
+            case PRE_GA_WRITE_ROWS_EVENT:
-            return "Table_map";
+                return "Write_rows_event_old";
-        case PRE_GA_WRITE_ROWS_EVENT:
+            case PRE_GA_UPDATE_ROWS_EVENT:
-            return "Write_rows_event_old";
+                return "Update_rows_event_old";
-        case PRE_GA_UPDATE_ROWS_EVENT:
+            case PRE_GA_DELETE_ROWS_EVENT:
-            return "Update_rows_event_old";
+                return "Delete_rows_event_old";
-        case PRE_GA_DELETE_ROWS_EVENT:
+            case WRITE_ROWS_EVENT_V1:
-            return "Delete_rows_event_old";
+                return "Write_rows_v1";
-        case WRITE_ROWS_EVENT_V1:
+            case UPDATE_ROWS_EVENT_V1:
-            return "Write_rows_v1";
+                return "Update_rows_v1";
-        case UPDATE_ROWS_EVENT_V1:
+            case DELETE_ROWS_EVENT_V1:
-            return "Update_rows_v1";
+                return "Delete_rows_v1";
-        case DELETE_ROWS_EVENT_V1:
+            case BEGIN_LOAD_QUERY_EVENT:
-            return "Delete_rows_v1";
+                return "Begin_load_query";
-        case BEGIN_LOAD_QUERY_EVENT:
+            case EXECUTE_LOAD_QUERY_EVENT:
-            return "Begin_load_query";
+                return "Execute_load_query";
-        case EXECUTE_LOAD_QUERY_EVENT:
+            case INCIDENT_EVENT:
-            return "Execute_load_query";
+                return "Incident";
-        case INCIDENT_EVENT:
+            case HEARTBEAT_LOG_EVENT:
-            return "Incident";
+                return "Heartbeat";
-        case HEARTBEAT_LOG_EVENT:
+            case IGNORABLE_LOG_EVENT:
-            return "Heartbeat";
+                return "Ignorable";
-        case IGNORABLE_LOG_EVENT:
+            case ROWS_QUERY_LOG_EVENT:
-            return "Ignorable";
+                return "Rows_query";
-        case ROWS_QUERY_LOG_EVENT:
+            case WRITE_ROWS_EVENT:
-            return "Rows_query";
+                return "Write_rows";
-        case WRITE_ROWS_EVENT:
+            case UPDATE_ROWS_EVENT:
-            return "Write_rows";
+                return "Update_rows";
-        case UPDATE_ROWS_EVENT:
+            case DELETE_ROWS_EVENT:
-            return "Update_rows";
+                return "Delete_rows";
-        case DELETE_ROWS_EVENT:
+            case GTID_LOG_EVENT:
-            return "Delete_rows";
+                return "Gtid";
-        case GTID_LOG_EVENT:
+            case ANONYMOUS_GTID_LOG_EVENT:
-            return "Gtid";
+                return "Anonymous_Gtid";
-        case ANONYMOUS_GTID_LOG_EVENT:
+            case PREVIOUS_GTIDS_LOG_EVENT:
-            return "Anonymous_Gtid";
+                return "Previous_gtids";
-        case PREVIOUS_GTIDS_LOG_EVENT:
+            default:
-            return "Previous_gtids";
+                return "Unknown"; /* impossible */
-        default:
-            return "Unknown"; /* impossible */
         }
     }
 
@@ -383,16 +360,14 @@ public abstract class LogEvent
 
     protected final LogHeader  header;
 
-    protected LogEvent(LogHeader header)
+    protected LogEvent(LogHeader header){
-    {
         this.header = header;
     }
 
     /**
      * Return event header.
      */
-    public final LogHeader getHeader()
+    public final LogHeader getHeader() {
-    {
         return header;
     }
 
@@ -400,16 +375,14 @@ public abstract class LogEvent
      * The total size of this event, in bytes. In other words, this is the sum
      * of the sizes of Common-Header, Post-Header, and Body.
      */
-    public final int getEventLen()
+    public final int getEventLen() {
-    {
         return header.getEventLen();
     }
 
     /**
      * Server ID of the server that created the event.
      */
-    public final long getServerId()
+    public final long getServerId() {
-    {
         return header.getServerId();
     }
 
@@ -420,18 +393,15 @@ public abstract class LogEvent
      * file. In a relay log, this is the position of the next event in the
      * master's binlog.
      */
-    public final long getLogPos()
+    public final long getLogPos() {
-    {
         return header.getLogPos();
     }
 
     /**
      * The time when the query started, in seconds since 1970.
      */
-    public final long getWhen()
+    public final long getWhen() {
-    {
         return header.getWhen();
     }
 
-    
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java

@@ -29,8 +29,8 @@ import java.util.Arrays;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class LogFetcher extends LogBuffer implements Closeable
+public abstract class LogFetcher extends LogBuffer implements Closeable {
-{
+
     /** Default initial capacity. */
     public static final int   DEFAULT_INITIAL_CAPACITY = 8192;
 
@@ -42,18 +42,15 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
 
     protected final float     factor;
 
-    public LogFetcher()
+    public LogFetcher(){
-    {
         this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
     }
 
-    public LogFetcher(final int initialCapacity)
+    public LogFetcher(final int initialCapacity){
-    {
         this(initialCapacity, DEFAULT_GROWTH_FACTOR);
     }
 
-    public LogFetcher(final int initialCapacity, final float growthFactor)
+    public LogFetcher(final int initialCapacity, final float growthFactor){
-    {
         this.buffer = new byte[initialCapacity];
         this.factor = growthFactor;
     }
@@ -65,15 +62,12 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
      * 
      * @param minCapacity the desired minimum capacity
      */
-    protected final void ensureCapacity(final int minCapacity)
+    protected final void ensureCapacity(final int minCapacity) {
-    {
         final int oldCapacity = buffer.length;
 
-        if (minCapacity > oldCapacity)
+        if (minCapacity > oldCapacity) {
-        {
             int newCapacity = (int) (oldCapacity * factor);
-            if (newCapacity < minCapacity)
+            if (newCapacity < minCapacity) newCapacity = minCapacity;
-                newCapacity = minCapacity;
 
             buffer = Arrays.copyOf(buffer, newCapacity);
         }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java

@@ -6,8 +6,8 @@ package com.taobao.tddl.dbsync.binlog;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class LogPosition implements Cloneable, Comparable<LogPosition>
+public class LogPosition implements Cloneable, Comparable<LogPosition> {
-{
+
     /* binlog file's name */
     protected String fileName;
 
@@ -19,8 +19,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @param fileName file name for binlog files: mysql-bin.000001
      */
-    public LogPosition(String fileName)
+    public LogPosition(String fileName){
-    {
         this.fileName = fileName;
         this.position = 0L;
     }
@@ -30,8 +29,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @param fileName file name for binlog files: mysql-bin.000001
      */
-    public LogPosition(String fileName, final long position)
+    public LogPosition(String fileName, final long position){
-    {
         this.fileName = fileName;
         this.position = position;
     }
@@ -39,31 +37,24 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
     /**
      * Binlog position copy init.
      */
-    public LogPosition(LogPosition source)
+    public LogPosition(LogPosition source){
-    {
         this.fileName = source.fileName;
         this.position = source.position;
     }
 
-    public final String getFileName()
+    public final String getFileName() {
-    {
         return fileName;
     }
 
-    public final long getPosition()
+    public final long getPosition() {
-    {
         return position;
     }
 
     /* Clone binlog position without CloneNotSupportedException */
-    public LogPosition clone()
+    public LogPosition clone() {
-    {
+        try {
-        try
-        {
             return (LogPosition) super.clone();
-        }
+        } catch (CloneNotSupportedException e) {
-        catch (CloneNotSupportedException e)
-        {
             // Never happend
             return null;
         }
@@ -72,12 +63,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
     /**
      * Compares with the specified fileName and position.
      */
-    public final int compareTo(String fileName, final long position)
+    public final int compareTo(String fileName, final long position) {
-    {
         final int val = this.fileName.compareTo(fileName);
 
-        if (val == 0)
+        if (val == 0) {
-        {
             return (int) (this.position - position);
         }
         return val;
@@ -88,12 +77,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Comparable#compareTo(java.lang.Object)
      */
-    public int compareTo(LogPosition o)
+    public int compareTo(LogPosition o) {
-    {
         final int val = fileName.compareTo(o.fileName);
 
-        if (val == 0)
+        if (val == 0) {
-        {
             return (int) (position - o.position);
         }
         return val;
@@ -104,13 +91,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Object#equals(java.lang.Object)
      */
-    public boolean equals(Object obj)
+    public boolean equals(Object obj) {
-    {
+        if (obj instanceof LogPosition) {
-        if (obj instanceof LogPosition)
-        {
             LogPosition pos = ((LogPosition) obj);
-            return fileName.equals(pos.fileName)
+            return fileName.equals(pos.fileName) && (this.position == pos.position);
-                    && (this.position == pos.position);
         }
         return false;
     }
@@ -120,8 +104,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Object#toString()
      */
-    public String toString()
+    public String toString() {
-    {
         return fileName + ':' + position;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java

@@ -9,8 +9,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class AppendBlockLogEvent extends LogEvent
+public class AppendBlockLogEvent extends LogEvent {
-{
+
     private final LogBuffer blockBuf;
     private final int       blockLen;
 
@@ -19,9 +19,7 @@ public class AppendBlockLogEvent extends LogEvent
     /* AB = "Append Block" */
     public static final int AB_FILE_ID_OFFSET = 0;
 
-    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
+    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -36,18 +34,15 @@ public class AppendBlockLogEvent extends LogEvent
         blockBuf = buffer.duplicate(blockLen);
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 
-    public final LogBuffer getBuffer()
+    public final LogBuffer getBuffer() {
-    {
         return blockBuf;
     }
 
-    public final byte[] getData()
+    public final byte[] getData() {
-    {
         return blockBuf.getData();
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java

@@ -10,11 +10,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
+public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent {
-{
+
-    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
+    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java

@@ -8,8 +8,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class CreateFileLogEvent extends LoadLogEvent
+public final class CreateFileLogEvent extends LoadLogEvent {
-{
+
     protected LogBuffer     blockBuf;
     protected int           blockLen;
     protected long          fileId;
@@ -20,9 +20,7 @@ public final class CreateFileLogEvent extends LoadLogEvent
     public static final int CF_FILE_ID_OFFSET = 0;
     public static final int CF_DATA_OFFSET    = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
 
-    public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
+    public CreateFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
 
         final int headerLen = descriptionEvent.commonHeaderLen;
@@ -30,44 +28,37 @@ public final class CreateFileLogEvent extends LoadLogEvent
         final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
 
         copyLogEvent(buffer,
-                ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
+            ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen) : (headerLen + loadHeaderLen + createFileHeaderLen)),
-                        : (headerLen + loadHeaderLen + createFileHeaderLen)),
+            descriptionEvent);
-                descriptionEvent);
 
-        if (descriptionEvent.binlogVersion != 1)
+        if (descriptionEvent.binlogVersion != 1) {
-        {
+            fileId = buffer.getUint32(headerLen + loadHeaderLen + CF_FILE_ID_OFFSET);
-            fileId = buffer.getUint32(headerLen + loadHeaderLen
-                    + CF_FILE_ID_OFFSET);
             /*
-              Note that it's ok to use get_data_size() below, because it is computed
+             * Note that it's ok to use get_data_size() below, because it is
-              with values we have already read from this event (because we called
+             * computed with values we have already read from this event
-              copy_log_event()); we are not using slave's format info to decode
+             * (because we called copy_log_event()); we are not using slave's
-              master's format, we are really using master's format info.
+             * format info to decode master's format, we are really using
-              Anyway, both formats should be identical (except the common_header_len)
+             * master's format info. Anyway, both formats should be identical
-              as these Load events are not changed between 4.0 and 5.0 (as logging of
+             * (except the common_header_len) as these Load events are not
-              LOAD DATA INFILE does not use Load_log_event in 5.0).
+             * changed between 4.0 and 5.0 (as logging of LOAD DATA INFILE does
-            */
+             * not use Load_log_event in 5.0).
+             */
             blockLen = buffer.limit() - buffer.position();
             blockBuf = buffer.duplicate(blockLen);
-        }
+        } else {
-        else
-        {
             initedFromOld = true;
         }
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 
-    public final LogBuffer getBuffer()
+    public final LogBuffer getBuffer() {
-    {
         return blockBuf;
     }
 
-    public final byte[] getData()
+    public final byte[] getData() {
-    {
         return blockBuf.getData();
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java

@@ -9,25 +9,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteFileLogEvent extends LogEvent
+public final class DeleteFileLogEvent extends LogEvent {
-{
+
     private final long      fileId;
 
     /* DF = "Delete File" */
     public static final int DF_FILE_ID_OFFSET = 0;
 
-    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
+    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET);
-        fileId = buffer.getUint32(); //  DF_FILE_ID_OFFSET
+        fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteRowsLogEvent extends RowsLogEvent
+public final class DeleteRowsLogEvent extends RowsLogEvent {
-{
+
-    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
+    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java

@@ -9,25 +9,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class ExecuteLoadLogEvent extends LogEvent
+public final class ExecuteLoadLogEvent extends LogEvent {
-{
+
     private final long      fileId;
 
     /* EL = "Execute Load" */
     public static final int EL_FILE_ID_OFFSET = 0;
 
-    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
+    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET);
-        fileId = buffer.getUint32(); //  EL_FILE_ID_OFFSET
+        fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java

@@ -8,7 +8,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * Event responsible for LOAD DATA execution, it similar to Query_log_event but
  * before executing the query it substitutes original filename in LOAD DATA
  * query with name of temporary file.
- * 
  * <ul>
  * <li>4 bytes. The ID of the file to load.</li>
  * <li>4 bytes. The start position within the statement for filename
@@ -22,8 +21,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
+public final class ExecuteLoadQueryLogEvent extends QueryLogEvent {
-{
+
     /** file_id of temporary file */
     private long            fileId;
 
@@ -53,9 +52,8 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
     public static final int ELQ_FN_POS_END_OFFSET   = ELQ_FILE_ID_OFFSET + 8;
     public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
 
-    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
+    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
-            FormatDescriptionLogEvent descriptionEvent) throws IOException
+                                                                                                                   throws IOException{
-    {
         super(header, buffer, descriptionEvent);
 
         buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
@@ -66,36 +64,27 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
         dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
 
         final int len = query.length();
-        if (fnPosStart > len || fnPosEnd > len
+        if (fnPosStart > len || fnPosEnd > len || dupHandling > LOAD_DUP_REPLACE) {
-                || dupHandling > LOAD_DUP_REPLACE)
+            throw new IOException(String.format("Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
-        {
+                                                + "fn_pos_end=%d, dup_handling=%d", fnPosStart, fnPosEnd, dupHandling));
-            throw new IOException(String.format(
-                    "Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
-                            + "fn_pos_end=%d, dup_handling=%d", fnPosStart,
-                    fnPosEnd, dupHandling));
         }
     }
 
-    public final int getFilenamePosStart()
+    public final int getFilenamePosStart() {
-    {
         return fnPosStart;
     }
 
-    public final int getFilenamePosEnd()
+    public final int getFilenamePosEnd() {
-    {
         return fnPosEnd;
     }
 
-    public final String getFilename()
+    public final String getFilename() {
-    {
+        if (query != null) return query.substring(fnPosStart, fnPosEnd).trim();
-        if (query != null)
-            return query.substring(fnPosStart, fnPosEnd).trim();
 
         return null;
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java

@@ -5,8 +5,8 @@ import java.io.IOException;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
 /**
- * For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
+ * For binlog version 4. This event is saved by threads which read it, as they
- * ordinary events).
+ * need it for future use (to decode the ordinary events).
  * 
  * @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
@@ -15,23 +15,24 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
     /**
-     * The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
+     * The number of types we handle in Format_description_log_event
-     * exist in binlogs, it does not have a format).
+     * (UNKNOWN_EVENT is not to be handled, it does not exist in binlogs, it
+     * does not have a format).
      */
-    public static final int    LOG_EVENT_TYPES                     = (ENUM_END_EVENT - 1);
+    public static final int   LOG_EVENT_TYPES                     = (ENUM_END_EVENT - 1);
 
-    public static final int    ST_COMMON_HEADER_LEN_OFFSET         = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
+    public static final int   ST_COMMON_HEADER_LEN_OFFSET         = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
 
-    public static final int    OLD_HEADER_LEN                      = 13;
+    public static final int   OLD_HEADER_LEN                      = 13;
-    public static final int    LOG_EVENT_HEADER_LEN                = 19;
+    public static final int   LOG_EVENT_HEADER_LEN                = 19;
-    public static final int    LOG_EVENT_MINIMAL_HEADER_LEN        = 19;
+    public static final int   LOG_EVENT_MINIMAL_HEADER_LEN        = 19;
 
     /* event-specific post-header sizes */
-    public static final int    STOP_HEADER_LEN                     = 0;
+    public static final int   STOP_HEADER_LEN                     = 0;
-    public static final int    LOAD_HEADER_LEN                     = (4 + 4 + 4 + 1 + 1 + 4);
+    public static final int   LOAD_HEADER_LEN                     = (4 + 4 + 4 + 1 + 1 + 4);
-    public static final int    SLAVE_HEADER_LEN                    = 0;
+    public static final int   SLAVE_HEADER_LEN                    = 0;
-    public static final int    START_V3_HEADER_LEN                 = (2 + ST_SERVER_VER_LEN + 4);
+    public static final int   START_V3_HEADER_LEN                 = (2 + ST_SERVER_VER_LEN + 4);
-    public static final int    ROTATE_HEADER_LEN                   = 8;                                                       // this
+    public static final int   ROTATE_HEADER_LEN                   = 8;                                                       // this
     // is
     // FROZEN
     // (the
@@ -39,47 +40,48 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
     // post-header
     // is
     // frozen)
-    public static final int    INTVAR_HEADER_LEN                   = 0;
+    public static final int   INTVAR_HEADER_LEN                   = 0;
-    public static final int    CREATE_FILE_HEADER_LEN              = 4;
+    public static final int   CREATE_FILE_HEADER_LEN              = 4;
-    public static final int    APPEND_BLOCK_HEADER_LEN             = 4;
+    public static final int   APPEND_BLOCK_HEADER_LEN             = 4;
-    public static final int    EXEC_LOAD_HEADER_LEN                = 4;
+    public static final int   EXEC_LOAD_HEADER_LEN                = 4;
-    public static final int    DELETE_FILE_HEADER_LEN              = 4;
+    public static final int   DELETE_FILE_HEADER_LEN              = 4;
-    public static final int    NEW_LOAD_HEADER_LEN                 = LOAD_HEADER_LEN;
+    public static final int   NEW_LOAD_HEADER_LEN                 = LOAD_HEADER_LEN;
-    public static final int    RAND_HEADER_LEN                     = 0;
+    public static final int   RAND_HEADER_LEN                     = 0;
-    public static final int    USER_VAR_HEADER_LEN                 = 0;
+    public static final int   USER_VAR_HEADER_LEN                 = 0;
-    public static final int    FORMAT_DESCRIPTION_HEADER_LEN       = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
+    public static final int   FORMAT_DESCRIPTION_HEADER_LEN       = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
-    public static final int    XID_HEADER_LEN                      = 0;
+    public static final int   XID_HEADER_LEN                      = 0;
-    public static final int    BEGIN_LOAD_QUERY_HEADER_LEN         = APPEND_BLOCK_HEADER_LEN;
+    public static final int   BEGIN_LOAD_QUERY_HEADER_LEN         = APPEND_BLOCK_HEADER_LEN;
-    public static final int    ROWS_HEADER_LEN_V1                  = 8;
+    public static final int   ROWS_HEADER_LEN_V1                  = 8;
-    public static final int    TABLE_MAP_HEADER_LEN                = 8;
+    public static final int   TABLE_MAP_HEADER_LEN                = 8;
-    public static final int    EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
+    public static final int   EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
-    public static final int    EXECUTE_LOAD_QUERY_HEADER_LEN       = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
+    public static final int   EXECUTE_LOAD_QUERY_HEADER_LEN       = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
-    public static final int    INCIDENT_HEADER_LEN                 = 2;
+    public static final int   INCIDENT_HEADER_LEN                 = 2;
-    public static final int    HEARTBEAT_HEADER_LEN                = 0;
+    public static final int   HEARTBEAT_HEADER_LEN                = 0;
-    public static final int    IGNORABLE_HEADER_LEN                = 0;
+    public static final int   IGNORABLE_HEADER_LEN                = 0;
-    public static final int    ROWS_HEADER_LEN_V2                  = 10;
+    public static final int   ROWS_HEADER_LEN_V2                  = 10;
-    public static final int    ANNOTATE_ROWS_HEADER_LEN            = 0;
+    public static final int   ANNOTATE_ROWS_HEADER_LEN            = 0;
-    public static final int    BINLOG_CHECKPOINT_HEADER_LEN        = 4;
+    public static final int   BINLOG_CHECKPOINT_HEADER_LEN        = 4;
-    public static final int    GTID_HEADER_LEN                     = 19;
+    public static final int   GTID_HEADER_LEN                     = 19;
-    public static final int    GTID_LIST_HEADER_LEN                = 4;
+    public static final int   GTID_LIST_HEADER_LEN                = 4;
 
-    public static final int    POST_HEADER_LENGTH                  = 11;
+    public static final int   POST_HEADER_LENGTH                  = 11;
 
     public static final int   BINLOG_CHECKSUM_ALG_DESC_LEN        = 1;
     public static final int[] checksumVersionSplit                = { 5, 6, 1 };
     public static final long  checksumVersionProduct              = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
                                                                     * 256 + checksumVersionSplit[2];
     /**
-     * The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
+     * The size of the fixed header which _all_ events have (for binlogs written
-     * LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
+     * by this version, this is equal to LOG_EVENT_HEADER_LEN), except
+     * FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
      * LOG_EVENT_MINIMAL_HEADER_LEN).
      */
-    protected final int        commonHeaderLen;
+    protected final int       commonHeaderLen;
-    protected int              numberOfEventTypes;
+    protected int             numberOfEventTypes;
 
     /** The list of post-headers' lengthes */
-    protected final short[]    postHeaderLen;
+    protected final short[]   postHeaderLen;
-    protected int[]            serverVersionSplit                  = new int[3];
+    protected int[]           serverVersionSplit                  = new int[3];
 
     public FormatDescriptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
                                                                                                                     throws IOException{
@@ -105,12 +107,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         calcServerVersionSplit();
         long calc = getVersionProduct();
         if (calc >= checksumVersionProduct) {
-            /* the last bytes are the checksum alg desc and value (or value's room) */
+            /*
+             * the last bytes are the checksum alg desc and value (or value's
+             * room)
+             */
             numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
         }
 
         if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= "
-                + numberOfEventTypes);
+                                                + numberOfEventTypes);
     }
 
     /** MySQL 5.0 format descriptions. */
@@ -136,6 +141,11 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         }
     }
 
+    public FormatDescriptionLogEvent(final int binlogVersion, int binlogChecksum){
+        this(binlogVersion);
+        this.header.checksumAlg = binlogChecksum;
+    }
+
     public FormatDescriptionLogEvent(final int binlogVersion){
         this.binlogVersion = binlogVersion;
 
@@ -147,7 +157,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 commonHeaderLen = LOG_EVENT_HEADER_LEN;
                 numberOfEventTypes = LOG_EVENT_TYPES;
 
-                /* Note: all event types must explicitly fill in their lengths here. */
+                /*
+                 * Note: all event types must explicitly fill in their lengths
+                 * here.
+                 */
                 postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
                 postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
                 postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
@@ -171,11 +184,14 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 /*
-                 * We here have the possibility to simulate a master of before we changed the table map id to be stored
+                 * We here have the possibility to simulate a master of before
-                 * in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
+                 * we changed the table map id to be stored in 6 bytes: when it
-                 * compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
+                 * was stored in 4 bytes (=> post_header_len was 6). This is
-                 * that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
+                 * used to test backward compatibility. This code can be removed
-                 * test (rpl_row_4_bytes) too.
+                 * after a few months (today is Dec 21st 2005), when we know
+                 * that the 4-byte masters are not deployed anymore (check with
+                 * Tomas Ulin first!), and the accompanying test
+                 * (rpl_row_4_bytes) too.
                  */
                 postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
                 postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
@@ -195,17 +211,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
             case 3: /* 4.0.x x>=2 */
                 /*
-                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
+                 * We build an artificial (i.e. not sent by the master) event,
-                 * versions send.
+                 * which describes what those old master versions send.
                  */
                 serverVersion = "4.0";
                 commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
 
                 /*
-                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
+                 * The first new event in binlog version 4 is Format_desc. So
-                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
+                 * any event type after that does not exist in older versions.
-                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
+                 * We use the events known by version 3, even if version 1 had
-                 * detect less corruptions).
+                 * only a subset of them (this is not a problem: it uses a few
+                 * bytes for nothing but unifies code; it does not make the
+                 * slave detect less corruptions).
                  */
                 numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
 
@@ -222,17 +240,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
             case 1: /* 3.23 */
                 /*
-                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
+                 * We build an artificial (i.e. not sent by the master) event,
-                 * versions send.
+                 * which describes what those old master versions send.
                  */
                 serverVersion = "3.23";
                 commonHeaderLen = OLD_HEADER_LEN;
 
                 /*
-                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
+                 * The first new event in binlog version 4 is Format_desc. So
-                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
+                 * any event type after that does not exist in older versions.
-                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
+                 * We use the events known by version 3, even if version 1 had
-                 * detect less corruptions).
+                 * only a subset of them (this is not a problem: it uses a few
+                 * bytes for nothing but unifies code; it does not make the
+                 * slave detect less corruptions).
                  */
                 numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java

@@ -6,19 +6,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * <pre>
  *   Base class for ignorable log events. Events deriving from
-  this class can be safely ignored by slaves that cannot
+ *   this class can be safely ignored by slaves that cannot
-  recognize them. Newer slaves, will be able to read and
+ *   recognize them. Newer slaves, will be able to read and
-  handle them. This has been designed to be an open-ended
+ *   handle them. This has been designed to be an open-ended
-  architecture, so adding new derived events shall not harm
+ *   architecture, so adding new derived events shall not harm
-  the old slaves that support ignorable log event mechanism
+ *   the old slaves that support ignorable log event mechanism
-  (they will just ignore unrecognized ignorable events).
+ *   (they will just ignore unrecognized ignorable events).
-
+ * 
-  @note The only thing that makes an event ignorable is that it has
+ *   @note The only thing that makes an event ignorable is that it has
-  the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
+ *   the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
-  that ignorable event types derive from Ignorable_log_event; they may
+ *   that ignorable event types derive from Ignorable_log_event; they may
-  just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
+ *   just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
-  argument to the Log_event constructor.
+ *   argument to the Log_event constructor.
-  </pre>
+ * </pre>
  * 
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java

@@ -5,12 +5,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * Class representing an incident, an occurance out of the ordinary, that
- * happened on the master.
+ * happened on the master. The event is used to inform the slave that something
- * 
+ * out of the ordinary happened on the master that might cause the database to
- * The event is used to inform the slave that something out of the ordinary
+ * be in an inconsistent state.
- * happened on the master that might cause the database to be in an inconsistent
- * state.
- * 
  * <table id="IncidentFormat">
  * <caption>Incident event format</caption>
  * <tr>
@@ -38,8 +35,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class IncidentLogEvent extends LogEvent
+public final class IncidentLogEvent extends LogEvent {
-{
+
     public static final int INCIDENT_NONE        = 0;
 
     /** There are possibly lost events in the replication stream */
@@ -51,9 +48,7 @@ public final class IncidentLogEvent extends LogEvent
     private final int       incident;
     private final String    message;
 
-    public IncidentLogEvent(LogHeader header, LogBuffer buffer,
+    public IncidentLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -61,10 +56,9 @@ public final class IncidentLogEvent extends LogEvent
 
         buffer.position(commonHeaderLen);
         final int incidentNumber = buffer.getUint16();
-        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
+        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE) {
-        {
             // If the incident is not recognized, this binlog event is
-            // invalid.  If we set incident_number to INCIDENT_NONE, the
+            // invalid. If we set incident_number to INCIDENT_NONE, the
             // invalidity will be detected by is_valid().
             incident = INCIDENT_NONE;
             message = null;
@@ -76,13 +70,11 @@ public final class IncidentLogEvent extends LogEvent
         message = buffer.getString();
     }
 
-    public final int getIncident()
+    public final int getIncident() {
-    {
         return incident;
     }
 
-    public final String getMessage()
+    public final String getMessage() {
-    {
         return message;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java

@@ -6,54 +6,43 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * An Intvar_log_event will be created just before a Query_log_event, if the
  * query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
- * Intvar_log_event holds the value of one of these variables.
+ * Intvar_log_event holds the value of one of these variables. Binary Format The
- * 
+ * Post-Header for this event type is empty. The Body has two components:
- * Binary Format
- * 
- * The Post-Header for this event type is empty. The Body has two components:
- * 
  * <table>
  * <caption>Body for Intvar_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>type</td>
  * <td>1 byte enumeration</td>
  * <td>One byte identifying the type of variable stored. Currently, two
  * identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
  * </tr>
- * 
  * <tr>
  * <td>value</td>
  * <td>8 byte unsigned integer</td>
  * <td>The value of the variable.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class IntvarLogEvent extends LogEvent
+public final class IntvarLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
      * 1 or INSERT_ID_EVENT = 2.</li>
      * <li>8 bytes. An unsigned integer indicating the value to be used for the
      * LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long      value;
@@ -68,31 +57,25 @@ public final class IntvarLogEvent extends LogEvent
     public static final int LAST_INSERT_ID_EVENT = 1;
     public static final int INSERT_ID_EVENT      = 2;
 
-    public IntvarLogEvent(LogHeader header, LogBuffer buffer,
+    public IntvarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         /* The Post-Header is empty. The Varible Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
-                + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+                        + I_TYPE_OFFSET);
-                + I_TYPE_OFFSET);
         type = buffer.getInt8(); // I_TYPE_OFFSET
         value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
     }
 
-    public final int getType()
+    public final int getType() {
-    {
         return type;
     }
 
-    public final long getValue()
+    public final long getValue() {
-    {
         return value;
     }
 
-    public final String getQuery()
+    public final String getQuery() {
-    {
         return "SET INSERT_ID = " + value;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java

@@ -26,16 +26,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * </pre>
  * 
  * Binary Format: The Post-Header consists of the following six components.
- * 
  * <table>
  * <caption>Post-Header for Load_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>slave_proxy_id</td>
  * <td>4 byte unsigned integer</td>
@@ -45,71 +42,57 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * temporary table local to the client. The slave_proxy_id is used to
  * distinguish temporary tables that belong to different clients.</td>
  * </tr>
- * 
  * <tr>
  * <td>exec_time</td>
  * <td>4 byte unsigned integer</td>
  * <td>The time from when the query started to when it was logged in the binlog,
  * in seconds.</td>
  * </tr>
- * 
  * <tr>
  * <td>skip_lines</td>
  * <td>4 byte unsigned integer</td>
  * <td>The number on line (14) above, if present, or 0 if line (14) is left out.
  * </td>
  * </tr>
- * 
  * <tr>
  * <td>table_name_len</td>
  * <td>1 byte unsigned integer</td>
  * <td>The length of 'table_name' on line (4) above.</td>
  * </tr>
- * 
  * <tr>
  * <td>db_len</td>
  * <td>1 byte unsigned integer</td>
  * <td>The length of 'db' on line (1) above.</td>
  * </tr>
- * 
  * <tr>
  * <td>num_fields</td>
  * <td>4 byte unsigned integer</td>
  * <td>The number n of fields on line (15) above.</td>
  * </tr>
  * </table>
- * 
  * The Body contains the following components.
- * 
  * <table>
  * <caption>Body of Load_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>sql_ex</td>
  * <td>variable length</td>
- * 
  * <td>Describes the part of the query on lines (3) and (5)&ndash;(13) above.
  * More precisely, it stores the five strings (on lines) field_term (6),
  * enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
  * bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
- * OPTIONALLY (7).
+ * OPTIONALLY (7). The data is stored in one of two formats, called "old" and
- * 
+ * "new". The type field of Common-Header determines which of these two formats
- * The data is stored in one of two formats, called "old" and "new". The type
+ * is used: type LOAD_EVENT means that the old format is used, and type
- * field of Common-Header determines which of these two formats is used: type
+ * NEW_LOAD_EVENT means that the new format is used. When MySQL writes a
- * LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
+ * Load_log_event, it uses the new format if at least one of the five strings is
- * that the new format is used. When MySQL writes a Load_log_event, it uses the
+ * two or more bytes long. Otherwise (i.e., if all strings are 0 or 1 bytes
- * new format if at least one of the five strings is two or more bytes long.
+ * long), the old format is used. The new and old format differ in the way the
- * Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
+ * five strings are stored.
- * used.
- * 
- * The new and old format differ in the way the five strings are stored.
- * 
  * <ul>
  * <li>In the new format, the strings are stored in the order field_term,
  * enclosed, escaped, line_term, line_start. Each string consists of a length (1
@@ -117,7 +100,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
  * IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
  * the presence of the corresponding keyword in the SQL query.
- * 
  * <li>In the old format, we know that each string has length 0 or 1. Therefore,
  * only the first byte of each string is stored. The order of the strings is the
  * same as in the new format. These five bytes are followed by the same 1 byte
@@ -128,18 +110,15 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
  * LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
  * </ul>
- * 
  * Thus, the size of the new format is 6 bytes + the sum of the sizes of the
  * five strings. The size of the old format is always 7 bytes.</td>
  * </tr>
- * 
  * <tr>
  * <td>field_lens</td>
  * <td>num_fields 1 byte unsigned integers</td>
  * <td>An array of num_fields integers representing the length of each field in
  * the query. (num_fields is from the Post-Header).</td>
  * </tr>
- * 
  * <tr>
  * <td>fields</td>
  * <td>num_fields null-terminated strings</td>
@@ -148,7 +127,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * the num_fields array.) The total length of all strings equals to the sum of
  * all field_lens, plus num_fields bytes for all the trailing zeros.</td>
  * </tr>
- * 
  * <tr>
  * <td>table_name</td>
  * <td>null-terminated string of length table_len+1 bytes</td>
@@ -156,31 +134,27 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * trailing zero is actually redundant since the table_len is known from
  * Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>db</td>
  * <td>null-terminated string of length db_len+1 bytes</td>
  * <td>The 'db' from the query, as a null-terminated string. (The trailing zero
  * is actually redundant since the db_len is known from Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>file_name</td>
  * <td>variable length string without trailing zero, extending to the end of the
  * event (determined by the length field of the Common-Header)</td>
  * <td>The 'file_name' from the query.</td>
  * </tr>
- * 
  * </table>
- * 
  * This event type is understood by current versions, but only generated by
  * MySQL 3.23 and earlier.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class LoadLogEvent extends LogEvent
+public class LoadLogEvent extends LogEvent {
-{
+
     private String          table;
     private String          db;
     private String          fname;
@@ -210,11 +184,10 @@ public class LoadLogEvent extends LogEvent
     public static final int L_DATA_OFFSET       = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
 
     /*
-      These are flags and structs to handle all the LOAD DATA INFILE options (LINES
+     * These are flags and structs to handle all the LOAD DATA INFILE options
-      TERMINATED etc).
+     * (LINES TERMINATED etc). DUMPFILE_FLAG is probably useless (DUMPFILE is a
-      DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
+     * clause of SELECT, not of LOAD DATA).
-      DATA).
+     */
-    */
     public static final int DUMPFILE_FLAG       = 0x1;
     public static final int OPT_ENCLOSED_FLAG   = 0x2;
     public static final int REPLACE_FLAG        = 0x4;
@@ -226,9 +199,7 @@ public class LoadLogEvent extends LogEvent
     public static final int LINE_START_EMPTY    = 0x8;
     public static final int ESCAPED_EMPTY       = 0x10;
 
-    public LoadLogEvent(LogHeader header, LogBuffer buffer,
+    public LoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
@@ -236,18 +207,16 @@ public class LoadLogEvent extends LogEvent
          * I (Guilhem) manually tested replication of LOAD DATA INFILE for
          * 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
          */
-        copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
+        copyLogEvent(buffer,
-                + descriptionEvent.commonHeaderLen : loadHeaderLen
+            ((header.type == LOAD_EVENT) ? loadHeaderLen + descriptionEvent.commonHeaderLen : loadHeaderLen
-                + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
+                                                                                              + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
-                descriptionEvent);
+            descriptionEvent);
     }
 
     /**
      * @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
      */
-    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
+    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset, FormatDescriptionLogEvent descriptionEvent) {
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         /* this is the beginning of the post-header */
         buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
 
@@ -262,8 +231,7 @@ public class LoadLogEvent extends LogEvent
          * Sql_ex.init() on success returns the pointer to the first byte after
          * the sql_ex structure, which is the start of field lengths array.
          */
-        if (header.type != LOAD_EVENT /* use_new_format */)
+        if (header.type != LOAD_EVENT /* use_new_format */) {
-        {
             /*
              * The code below assumes that buf will not disappear from under our
              * feet during the lifetime of the event. This assumption holds true
@@ -279,9 +247,7 @@ public class LoadLogEvent extends LogEvent
             escaped = buffer.getString();
             optFlags = buffer.getInt8();
             emptyFlags = 0;
-        }
+        } else {
-        else
-        {
             fieldTerm = buffer.getFixString(1);
             enclosed = buffer.getFixString(1);
             lineTerm = buffer.getFixString(1);
@@ -290,23 +256,17 @@ public class LoadLogEvent extends LogEvent
             optFlags = buffer.getUint8();
             emptyFlags = buffer.getUint8();
 
-            if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
+            if ((emptyFlags & FIELD_TERM_EMPTY) != 0) fieldTerm = null;
-                fieldTerm = null;
+            if ((emptyFlags & ENCLOSED_EMPTY) != 0) enclosed = null;
-            if ((emptyFlags & ENCLOSED_EMPTY) != 0)
+            if ((emptyFlags & LINE_TERM_EMPTY) != 0) lineTerm = null;
-                enclosed = null;
+            if ((emptyFlags & LINE_START_EMPTY) != 0) lineStart = null;
-            if ((emptyFlags & LINE_TERM_EMPTY) != 0)
+            if ((emptyFlags & ESCAPED_EMPTY) != 0) escaped = null;
-                lineTerm = null;
-            if ((emptyFlags & LINE_START_EMPTY) != 0)
-                lineStart = null;
-            if ((emptyFlags & ESCAPED_EMPTY) != 0)
-                escaped = null;
         }
 
         final int fieldLenPos = buffer.position();
         buffer.forward(numFields);
         fields = new String[numFields];
-        for (int i = 0; i < numFields; i++)
+        for (int i = 0; i < numFields; i++) {
-        {
             final int fieldLen = buffer.getUint8(fieldLenPos + i);
             fields[i] = buffer.getFixString(fieldLen + 1);
         }
@@ -324,68 +284,55 @@ public class LoadLogEvent extends LogEvent
         buffer.forward(1); // The + 1 is for \0 terminating fname
     }
 
-    public final String getTable()
+    public final String getTable() {
-    {
         return table;
     }
 
-    public final String getDb()
+    public final String getDb() {
-    {
         return db;
     }
 
-    public final String getFname()
+    public final String getFname() {
-    {
         return fname;
     }
 
-    public final int getSkipLines()
+    public final int getSkipLines() {
-    {
         return skipLines;
     }
 
-    public final String[] getFields()
+    public final String[] getFields() {
-    {
         return fields;
     }
 
-    public final String getFieldTerm()
+    public final String getFieldTerm() {
-    {
         return fieldTerm;
     }
 
-    public final String getLineTerm()
+    public final String getLineTerm() {
-    {
         return lineTerm;
     }
 
-    public final String getLineStart()
+    public final String getLineStart() {
-    {
         return lineStart;
     }
 
-    public final String getEnclosed()
+    public final String getEnclosed() {
-    {
         return enclosed;
     }
 
-    public final String getEscaped()
+    public final String getEscaped() {
-    {
         return escaped;
     }
 
-    public final int getOptFlags()
+    public final int getOptFlags() {
-    {
         return optFlags;
     }
 
-    public final int getEmptyFlags()
+    public final int getEmptyFlags() {
-    {
         return emptyFlags;
     }
 
-    public final long getExecTime()
+    public final long getExecTime() {
-    {
         return execTime;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java

@@ -11,41 +11,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * different format and length even for different events of the same type. The
  * binary formats of Post-Header and Body are documented separately in each
  * subclass. The binary format of Common-Header is as follows.
- * 
  * <table>
  * <caption>Common-Header</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>timestamp</td>
  * <td>4 byte unsigned integer</td>
  * <td>The time when the query started, in seconds since 1970.</td>
  * </tr>
- * 
  * <tr>
  * <td>type</td>
  * <td>1 byte enumeration</td>
  * <td>See enum #Log_event_type.</td>
  * </tr>
- * 
  * <tr>
  * <td>server_id</td>
  * <td>4 byte unsigned integer</td>
  * <td>Server ID of the server that created the event.</td>
  * </tr>
- * 
  * <tr>
  * <td>total_size</td>
  * <td>4 byte unsigned integer</td>
  * <td>The total size of this event, in bytes. In other words, this is the sum
  * of the sizes of Common-Header, Post-Header, and Body.</td>
  * </tr>
- * 
  * <tr>
  * <td>master_position</td>
  * <td>4 byte unsigned integer</td>
@@ -54,24 +47,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * the position of the next event, in bytes from the beginning of the file. In a
  * relay log, this is the position of the next event in the master's binlog.</td>
  * </tr>
- * 
  * <tr>
  * <td>flags</td>
  * <td>2 byte bitfield</td>
  * <td>See Log_event::flags.</td>
  * </tr>
  * </table>
- * 
  * Summing up the numbers above, we see that the total size of the common header
  * is 19 bytes.
  * 
  * @see mysql-5.1.60/sql/log_event.cc
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogHeader
+public final class LogHeader {
-{
+
     protected final int type;
 
     /**
@@ -114,34 +104,29 @@ public final class LogHeader
 
     /**
      * The value is set by caller of FD constructor and
-     * Log_event::write_header() for the rest.
+     * Log_event::write_header() for the rest. In the FD case it's propagated
-     * In the FD case it's propagated into the last byte 
+     * into the last byte of post_header_len[] at FD::write(). On the slave side
-     * of post_header_len[] at FD::write().
+     * the value is assigned from post_header_len[last] of the last seen FD
-     * On the slave side the value is assigned from post_header_len[last] 
+     * event.
-     * of the last seen FD event.
      */
     protected int       checksumAlg;
     /**
-       Placeholder for event checksum while writing to binlog.
+     * Placeholder for event checksum while writing to binlog.
-    */
+     */
-    protected long      crc;      // ha_checksum
+    protected long      crc;        // ha_checksum
 
     /* for Start_event_v3 */
-    public LogHeader(final int type)
+    public LogHeader(final int type){
-    {
         this.type = type;
     }
 
-    public LogHeader(LogBuffer buffer,
+    public LogHeader(LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         when = buffer.getUint32();
         type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET;
         serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET;
         eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET;
 
-        if (descriptionEvent.binlogVersion == 1)
+        if (descriptionEvent.binlogVersion == 1) {
-        {
             logPos = 0;
             flags = 0;
             return;
@@ -150,49 +135,44 @@ public final class LogHeader
         /* 4.0 or newer */
         logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET
         /*
-          If the log is 4.0 (so here it can only be a 4.0 relay log read by
+         * If the log is 4.0 (so here it can only be a 4.0 relay log read by the
-          the SQL thread or a 4.0 master binlog read by the I/O thread),
+         * SQL thread or a 4.0 master binlog read by the I/O thread), log_pos is
-          log_pos is the beginning of the event: we transform it into the end
+         * the beginning of the event: we transform it into the end of the
-          of the event, which is more useful.
+         * event, which is more useful. But how do you know that the log is 4.0:
-          But how do you know that the log is 4.0: you know it if
+         * you know it if description_event is version 3 *and* you are not
-          description_event is version 3 *and* you are not reading a
+         * reading a Format_desc (remember that mysqlbinlog starts by assuming
-          Format_desc (remember that mysqlbinlog starts by assuming that 5.0
+         * that 5.0 logs are in 4.0 format, until it finds a Format_desc).
-          logs are in 4.0 format, until it finds a Format_desc).
+         */
-        */
+        if (descriptionEvent.binlogVersion == 3 && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0) {
-        if (descriptionEvent.binlogVersion == 3
-                && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0)
-        {
             /*
-              If log_pos=0, don't change it. log_pos==0 is a marker to mean
+             * If log_pos=0, don't change it. log_pos==0 is a marker to mean
-              "don't change rli->group_master_log_pos" (see
+             * "don't change rli->group_master_log_pos" (see
-              inc_group_relay_log_pos()). As it is unreal log_pos, adding the
+             * inc_group_relay_log_pos()). As it is unreal log_pos, adding the
-              event len's is nonsense. For example, a fake Rotate event should
+             * event len's is nonsense. For example, a fake Rotate event should
-              not have its log_pos (which is 0) changed or it will modify
+             * not have its log_pos (which is 0) changed or it will modify
-              Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
+             * Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
-              value of (a non-zero offset which does not exist in the master's
+             * value of (a non-zero offset which does not exist in the master's
-              binlog, so which will cause problems if the user uses this value
+             * binlog, so which will cause problems if the user uses this value
-              in CHANGE MASTER).
+             * in CHANGE MASTER).
-            */
+             */
             logPos += eventLen; /* purecov: inspected */
         }
 
         flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET
-        if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT)
+        if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT) || (type == LogEvent.ROTATE_EVENT)) {
-                || (type == LogEvent.ROTATE_EVENT))
-        {
             /*
-              These events always have a header which stops here (i.e. their
+             * These events always have a header which stops here (i.e. their
-              header is FROZEN).
+             * header is FROZEN).
-            */
+             */
             /*
-              Initialization to zero of all other Log_event members as they're
+             * Initialization to zero of all other Log_event members as they're
-              not specified. Currently there are no such members; in the future
+             * not specified. Currently there are no such members; in the future
-              there will be an event UID (but Format_description and Rotate
+             * there will be an event UID (but Format_description and Rotate
-              don't need this UID, as they are not propagated through
+             * don't need this UID, as they are not propagated through
-              --log-slave-updates (remember the UID is used to not play a query
+             * --log-slave-updates (remember the UID is used to not play a query
-              twice when you have two masters which are slaves of a 3rd master).
+             * twice when you have two masters which are slaves of a 3rd
-              Then we are done.
+             * master). Then we are done.
-            */
+             */
 
             if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) {
                 int commonHeaderLen = buffer.getUint8(FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN
@@ -213,25 +193,22 @@ public final class LogHeader
         }
 
         /*
-        CRC verification by SQL and Show-Binlog-Events master side.
+         * CRC verification by SQL and Show-Binlog-Events master side. The
-        The caller has to provide @description_event->checksum_alg to
+         * caller has to provide @description_event->checksum_alg to be the last
-        be the last seen FD's (A) descriptor.
+         * seen FD's (A) descriptor. If event is FD the descriptor is in it.
-        If event is FD the descriptor is in it.
+         * Notice, FD of the binlog can be only in one instance and therefore
-        Notice, FD of the binlog can be only in one instance and therefore
+         * Show-Binlog-Events executing master side thread needs just to know
-        Show-Binlog-Events executing master side thread needs just to know
+         * the only FD's (A) value - whereas RL can contain more. In the RL
-        the only FD's (A) value -  whereas RL can contain more.
+         * case, the alg is kept in FD_e (@description_event) which is reset to
-        In the RL case, the alg is kept in FD_e (@description_event) which is reset 
+         * the newer read-out event after its execution with possibly new alg
-        to the newer read-out event after its execution with possibly new alg descriptor.
+         * descriptor. Therefore in a typical sequence of RL: {FD_s^0, FD_m,
-        Therefore in a typical sequence of RL:
+         * E_m^1} E_m^1 will be verified with (A) of FD_m. See legends
-        {FD_s^0, FD_m, E_m^1} E_m^1 
+         * definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs lines
-        will be verified with (A) of FD_m.
+         * (log.h). Notice, a pre-checksum FD version forces alg :=
-
+         * BINLOG_CHECKSUM_ALG_UNDEF.
-        See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs
+         */
-        lines (log.h).
+        checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch
-
+                                                                // checksum alg
-        Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF.
-        */
-        checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg
         processCheckSum(buffer);
         /* otherwise, go on with reading the header from buf (nothing now) */
     }
@@ -239,8 +216,7 @@ public final class LogHeader
     /**
      * The different types of log events.
      */
-    public final int getType()
+    public final int getType() {
-    {
         return type;
     }
 
@@ -251,8 +227,7 @@ public final class LogHeader
      * file. In a relay log, this is the position of the next event in the
      * master's binlog.
      */
-    public final long getLogPos()
+    public final long getLogPos() {
-    {
         return logPos;
     }
 
@@ -260,24 +235,21 @@ public final class LogHeader
      * The total size of this event, in bytes. In other words, this is the sum
      * of the sizes of Common-Header, Post-Header, and Body.
      */
-    public final int getEventLen()
+    public final int getEventLen() {
-    {
         return eventLen;
     }
 
     /**
      * The time when the query started, in seconds since 1970.
      */
-    public final long getWhen()
+    public final long getWhen() {
-    {
         return when;
     }
 
     /**
      * Server ID of the server that created the event.
      */
-    public final long getServerId()
+    public final long getServerId() {
-    {
         return serverId;
     }
 
@@ -286,25 +258,21 @@ public final class LogHeader
      * LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
      * LOG_EVENT_SUPPRESS_USE_F for notes.
      */
-    public final int getFlags()
+    public final int getFlags() {
-    {
         return flags;
     }
 
-    
     public long getCrc() {
         return crc;
     }
 
-    
     public int getChecksumAlg() {
         return checksumAlg;
     }
 
     private void processCheckSum(LogBuffer buffer) {
-        if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF &&
+        if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
-                checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){
+            crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
-            crc = buffer.getUint32(eventLen -  LogEvent.BINLOG_CHECKSUM_LEN);
         }
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java

@@ -4,16 +4,14 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * 
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3
  * @since mysql 5.6
  */
-public class PreviousGtidsLogEvent  extends LogEvent{
+public class PreviousGtidsLogEvent extends LogEvent {
 
     public PreviousGtidsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
         // do nothing , just for mysql gtid search function
     }
 }
-

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java

@@ -7,30 +7,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1
  * does not need it (it's repeatable again) so this event needn't be written in
  * 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it
- * does not cause bugs).
+ * does not cause bugs). The state of the random number generation consists of
- * 
+ * 128 bits, which are stored internally as two 64-bit numbers. Binary Format
- * The state of the random number generation consists of 128 bits, which are
- * stored internally as two 64-bit numbers.
- * 
- * Binary Format
- * 
  * The Post-Header for this event type is empty. The Body has two components:
- * 
  * <table>
  * <caption>Body for Rand_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>seed1</td>
  * <td>8 byte unsigned integer</td>
  * <td>64 bit random seed1.</td>
  * </tr>
- * 
  * <tr>
  * <td>seed2</td>
  * <td>8 byte unsigned integer</td>
@@ -41,19 +32,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class RandLogEvent extends LogEvent
+public final class RandLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>8 bytes. The value for the first seed.</li>
      * <li>8 bytes. The value for the second seed.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long      seed1;
@@ -63,21 +51,17 @@ public final class RandLogEvent extends LogEvent
     public static final int RAND_SEED1_OFFSET = 0;
     public static final int RAND_SEED2_OFFSET = 8;
 
-    public RandLogEvent(LogHeader header, LogBuffer buffer,
+    public RandLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         /* The Post-Header is empty. The Variable Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
-                + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
+                        + RAND_SEED1_OFFSET);
-                + RAND_SEED1_OFFSET);
         seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET);
         seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET);
     }
 
-    public final String getQuery()
+    public final String getQuery() {
-    {
         return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java

@@ -4,57 +4,44 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * This will be deprecated when we move to using sequence ids.
+ * This will be deprecated when we move to using sequence ids. Binary Format The
- * 
+ * Post-Header has one component:
- * Binary Format
- * 
- * The Post-Header has one component:
- * 
  * <table>
  * <caption>Post-Header for Rotate_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>position</td>
  * <td>8 byte integer</td>
  * <td>The position within the binlog to rotate to.</td>
  * </tr>
- * 
  * </table>
- * 
  * The Body has one component:
- * 
  * <table>
  * <caption>Body for Rotate_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>new_log</td>
  * <td>variable length string without trailing zero, extending to the end of the
  * event (determined by the length field of the Common-Header)</td>
  * <td>Name of the binlog to rotate to.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class RotateLogEvent extends LogEvent
+public final class RotateLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part:
-     * 
      * <ul>
      * <li>8 bytes. The position of the first event in the next log file. Always
      * contains the number 4 (meaning the next event starts at position 4 in the
@@ -62,14 +49,11 @@ public final class RotateLogEvent extends LogEvent
      * is assumed to be 4.</li>
      * </ul>
      * <p>
-     * 
      * Variable data part:
-     * 
      * <ul>
      * <li>The name of the next binary log. The filename is not null-terminated.
      * Its length is the event size minus the size of the fixed parts.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final String          filename;
@@ -91,22 +75,22 @@ public final class RotateLogEvent extends LogEvent
      * 
      * @throws MySQLExtractException
      */
-    public RotateLogEvent(LogHeader header, LogBuffer buffer,
+    public RotateLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int headerSize = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1];
 
         buffer.position(headerSize + R_POS_OFFSET);
-        position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET)
+        position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf
+                                                                  // +
+                                                                  // R_POS_OFFSET)
 
         final int filenameOffset = headerSize + postHeaderLen;
         int filenameLen = buffer.limit() - filenameOffset;
-        if (filenameLen > FN_REFLEN - 1)
+        if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1;
-            filenameLen = FN_REFLEN - 1;
         buffer.position(filenameOffset);
+
         filename = buffer.getFixString(filenameLen);
     }
 
@@ -114,8 +98,7 @@ public final class RotateLogEvent extends LogEvent
      * Creates a new <code>Rotate_log_event</code> without log information. This
      * is used to generate missing log rotation events.
      */
-    public RotateLogEvent(String filename)
+    public RotateLogEvent(String filename){
-    {
         super(ROTATE_HEADER);
 
         this.filename = filename;
@@ -125,21 +108,18 @@ public final class RotateLogEvent extends LogEvent
     /**
      * Creates a new <code>Rotate_log_event</code> without log information.
      */
-    public RotateLogEvent(String filename, final long position)
+    public RotateLogEvent(String filename, final long position){
-    {
         super(ROTATE_HEADER);
 
         this.filename = filename;
         this.position = position;
     }
 
-    public final String getFilename()
+    public final String getFilename() {
-    {
         return filename;
     }
 
-    public final long getPosition()
+    public final long getPosition() {
-    {
         return position;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java

@@ -8,6 +8,8 @@ import java.util.BitSet;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.taobao.tddl.dbsync.binlog.JsonConversion;
+import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
@@ -24,6 +26,8 @@ public final class RowsLogBuffer {
 
     public static final long   DATETIMEF_INT_OFS = 0x8000000000L;
     public static final long   TIMEF_INT_OFS     = 0x800000L;
+    public static final long   TIMEF_OFS         = 0x800000000000L;
+
     private final LogBuffer    buffer;
     private final int          columnLen;
     private final String       charsetName;
@@ -162,10 +166,13 @@ public final class RowsLogBuffer {
 
             case LogEvent.MYSQL_TYPE_TIMESTAMP:
             case LogEvent.MYSQL_TYPE_DATETIME:
+            case LogEvent.MYSQL_TYPE_TIMESTAMP2:
+            case LogEvent.MYSQL_TYPE_DATETIME2:
                 javaType = Types.TIMESTAMP;
                 break;
 
             case LogEvent.MYSQL_TYPE_TIME:
+            case LogEvent.MYSQL_TYPE_TIME2:
                 javaType = Types.TIME;
                 break;
 
@@ -437,14 +444,23 @@ public final class RowsLogBuffer {
                         break;
                 }
 
+                String second = null;
                 if (tv_sec == 0) {
-                    value = "0000-00-00 00:00:00";
+                    second = "0000-00-00 00:00:00";
                 } else {
                     Timestamp time = new Timestamp(tv_sec * 1000);
-                    time.setNanos(tv_usec * 1000);
+                    second = time.toString();
-                    String v = time.toString();
+                    second = second.substring(0, second.length() - 2);// 去掉毫秒精度.0
-                    value = v.substring(0, v.length() - 2);
                 }
+
+                if (meta >= 1) {
+                    String microSecond = usecondsToStr(tv_usec, meta);
+                    microSecond = microSecond.substring(0, meta);
+                    value = second + '.' + microSecond;
+                } else {
+                    value = second;
+                }
+
                 javaType = Types.TIMESTAMP;
                 length = 4 + (meta + 1) / 2;
                 break;
@@ -487,7 +503,6 @@ public final class RowsLogBuffer {
                  * .YYdddddh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff
                  */
                 long intpart = buffer.getBeUlong40() - DATETIMEF_INT_OFS; // big-endian
-                @SuppressWarnings("unused")
                 int frac = 0;
                 switch (meta) {
                     case 0:
@@ -510,8 +525,9 @@ public final class RowsLogBuffer {
                         break;
                 }
 
+                String second = null;
                 if (intpart == 0) {
-                    value = "0000-00-00 00:00:00";
+                    second = "0000-00-00 00:00:00";
                 } else {
                     // 构造TimeStamp只处理到秒
                     long ymd = intpart >> 17;
@@ -524,7 +540,7 @@ public final class RowsLogBuffer {
                     // % (1 << 5)), (int) (hms >> 12),
                     // (int) ((hms >> 6) % (1 << 6)), (int) (hms % (1 << 6)));
                     // value = new Timestamp(cal.getTimeInMillis());
-                    value = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                    second = String.format("%04d-%02d-%02d %02d:%02d:%02d",
                         (int) (ym / 13),
                         (int) (ym % 13),
                         (int) (ymd % (1 << 5)),
@@ -532,6 +548,15 @@ public final class RowsLogBuffer {
                         (int) ((hms >> 6) % (1 << 6)),
                         (int) (hms % (1 << 6)));
                 }
+
+                if (meta >= 1) {
+                    String microSecond = usecondsToStr(frac, meta);
+                    microSecond = microSecond.substring(0, meta);
+                    value = second + '.' + microSecond;
+                } else {
+                    value = second;
+                }
+
                 javaType = Types.TIMESTAMP;
                 length = 5 + (meta + 1) / 2;
                 break;
@@ -603,7 +628,8 @@ public final class RowsLogBuffer {
                             frac -= 0x100; /* -(0x100 - frac) */
                             // fraclong = frac * 10000;
                         }
-                        ltime = intpart << 24 + frac * 10000;
+                        frac = frac * 10000;
+                        ltime = intpart << 24;
                         break;
                     case 3:
                     case 4:
@@ -619,12 +645,14 @@ public final class RowsLogBuffer {
                             frac -= 0x10000; /* -(0x10000-frac) */
                             // fraclong = frac * 100;
                         }
-                        ltime = intpart << 24 + frac * 100;
+                        frac = frac * 100;
+                        ltime = intpart << 24;
                         break;
                     case 5:
                     case 6:
-                        intpart = buffer.getBeUlong48() - TIMEF_INT_OFS;
+                        intpart = buffer.getBeUlong48() - TIMEF_OFS;
                         ltime = intpart;
+                        frac = (int) (intpart % (1L << 24));
                         break;
                     default:
                         intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
@@ -632,8 +660,9 @@ public final class RowsLogBuffer {
                         break;
                 }
 
+                String second = null;
                 if (intpart == 0) {
-                    value = "00:00:00";
+                    second = "00:00:00";
                 } else {
                     // 目前只记录秒，不处理us frac
                     // if (cal == null) cal = Calendar.getInstance();
@@ -644,13 +673,21 @@ public final class RowsLogBuffer {
                     // value = new Time(cal.getTimeInMillis());
                     long ultime = Math.abs(ltime);
                     intpart = ultime >> 24;
-                    value = String.format("%s%02d:%02d:%02d",
+                    second = String.format("%s%02d:%02d:%02d",
                         ltime >= 0 ? "" : "-",
                         (int) ((intpart >> 12) % (1 << 10)),
                         (int) ((intpart >> 6) % (1 << 6)),
                         (int) (intpart % (1 << 6)));
                 }
 
+                if (meta >= 1) {
+                    String microSecond = usecondsToStr(Math.abs(frac), meta);
+                    microSecond = microSecond.substring(0, meta);
+                    value = second + '.' + microSecond;
+                } else {
+                    value = second;
+                }
+
                 javaType = Types.TIME;
                 length = 3 + (meta + 1) / 2;
                 break;
@@ -911,6 +948,42 @@ public final class RowsLogBuffer {
                 length = len;
                 break;
             }
+            case LogEvent.MYSQL_TYPE_JSON: {
+                switch (meta) {
+                    case 1: {
+                        len = buffer.getUint8();
+                        break;
+                    }
+                    case 2: {
+                        len = buffer.getUint16();
+                        break;
+                    }
+                    case 3: {
+                        len = buffer.getUint24();
+                        break;
+                    }
+                    case 4: {
+                        len = (int) buffer.getUint32();
+                        break;
+                    }
+                    default:
+                        throw new IllegalArgumentException("!! Unknown JSON packlen = " + meta);
+                }
+                // len = buffer.getUint16();
+                // buffer.forward(meta - 4);
+                int position = buffer.position();
+                Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName);
+                StringBuilder builder = new StringBuilder();
+                jsonValue.toJsonString(builder, charsetName);
+                value = builder.toString();
+                buffer.position(position + len);
+                // byte[] binary = new byte[len];
+                // buffer.fillBytes(binary, 0, len);
+                // value = binary;
+                javaType = Types.VARCHAR;
+                length = len;
+                break;
+            }
             case LogEvent.MYSQL_TYPE_GEOMETRY: {
                 /*
                  * MYSQL_TYPE_GEOMETRY: copy from BLOB or TEXT
@@ -973,4 +1046,24 @@ public final class RowsLogBuffer {
     public final int getLength() {
         return length;
     }
+
+    private String usecondsToStr(int frac, int meta) {
+        String sec = String.valueOf(frac);
+        if (meta > 6) {
+            throw new IllegalArgumentException("unknow useconds meta : " + meta);
+        }
+
+        if (sec.length() < 6) {
+            StringBuilder result = new StringBuilder(6);
+            int len = 6 - sec.length();
+            for (; len > 0; len--) {
+                result.append('0');
+            }
+            result.append(sec);
+            sec = result.toString();
+        }
+
+        return sec.substring(0, meta);
+    }
+
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java

@@ -12,41 +12,33 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class RowsLogEvent extends LogEvent
+public abstract class RowsLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part:
-     * 
      * <ul>
      * <li>6 bytes. The table ID.</li>
      * <li>2 bytes. Reserved for future use.</li>
      * </ul>
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>Packed integer. The number of columns in the table.</li>
-     * 
      * <li>Variable-sized. Bit-field indicating whether each column is used, one
      * bit per column. For this field, the amount of storage required for N
      * columns is INT((N+7)/8) bytes.</li>
-     * 
      * <li>Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating
      * whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one
      * bit per column. For this field, the amount of storage required for N
      * columns is INT((N+7)/8) bytes.</li>
-     * 
      * <li>Variable-sized. A sequence of zero or more rows. The end is
      * determined by the size of the event. Each row has the following format:
-     * 
      * <ul>
      * <li>Variable-sized. Bit-field indicating whether each field in the row is
      * NULL. Only columns that are "used" according to the second field in the
      * variable data part are listed here. If the second field in the variable
      * data part has N one-bits, the amount of storage required for this field
      * is INT((N+7)/8) bytes.</li>
-     * 
      * <li>Variable-sized. The row-image, containing values of all table fields.
      * This only lists table fields that are used (according to the second field
      * of the variable data part) and non-NULL (according to the previous
@@ -54,16 +46,18 @@ public abstract class RowsLogEvent extends LogEvent
      * number of zero bits in the previous field (not counting padding bits in
      * the last byte). The format of each value is described in the
      * log_event_print_value() function in log_event.cc.</li>
-     * 
      * <li>(for UPDATE_ROWS_EVENT only) the previous two fields are repeated,
      * representing a second table row.</li>
      * </ul>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long       tableId;                           /* Table ID */
-    private TableMapLogEvent table;                             /* The table the rows belong to */
+    private TableMapLogEvent table;                             /*
+                                                                  * The table
+                                                                  * the rows
+                                                                  * belong to
+                                                                  */
 
     /** Bitmap denoting columns available */
     protected final int      columnLen;
@@ -78,15 +72,17 @@ public abstract class RowsLogEvent extends LogEvent
     protected final BitSet   changeColumns;
 
     /** XXX: Don't handle buffer in another thread. */
-    private final LogBuffer  rowsBuf;                           /* The rows in packed format */
+    private final LogBuffer  rowsBuf;                           /*
+                                                                  * The rows in
+                                                                  * packed
+                                                                  * format
+                                                                  */
 
     /**
-     * enum enum_flag
+     * enum enum_flag These definitions allow you to combine the flags into an
-     * 
+     * appropriate flag set using the normal bitwise operators. The implicit
-     * These definitions allow you to combine the flags into an appropriate flag
+     * conversion from an enum-constant to an integer is accepted by the
-     * set using the normal bitwise operators. The implicit conversion from an
+     * compiler, which is then used to set the real set of flags.
-     * enum-constant to an integer is accepted by the compiler, which is then
-     * used to set the real set of flags.
      */
     private final int        flags;
 
@@ -112,43 +108,40 @@ public abstract class RowsLogEvent extends LogEvent
     public static final int  RW_V_TAG_LEN            = 1;
     public static final int  RW_V_EXTRAINFO_TAG      = 0;
 
-
+    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-    public RowsLogEvent(LogHeader header, LogBuffer buffer,
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
         int headerLen = 0;
         buffer.position(commonHeaderLen + RW_MAPID_OFFSET);
-        if (postHeaderLen == 6)
+        if (postHeaderLen == 6) {
-        {
+            /*
-            /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
+             * Master is of an intermediate source tree before 5.1.4. Id is 4
+             * bytes
+             */
             tableId = buffer.getUint32();
-        }
+        } else {
-        else
-        {
             tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET
         }
         flags = buffer.getUint16();
 
-        if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2)
+        if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2) {
-        {
             headerLen = buffer.getUint16();
             headerLen -= 2;
             int start = buffer.position();
             int end = start + headerLen;
-            for(int i = start ;i < end; ){
+            for (int i = start; i < end;) {
                 switch (buffer.getUint8(i++)) {
                     case RW_V_EXTRAINFO_TAG:
                         // int infoLen = buffer.getUint8();
                         buffer.position(i + EXTRA_ROW_INFO_LEN_OFFSET);
                         int checkLen = buffer.getUint8(); // EXTRA_ROW_INFO_LEN_OFFSET
-                        int val= checkLen - EXTRA_ROW_INFO_HDR_BYTES;
+                        int val = checkLen - EXTRA_ROW_INFO_HDR_BYTES;
-                        assert(buffer.getUint8() == val); //EXTRA_ROW_INFO_FORMAT_OFFSET
+                        assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_FORMAT_OFFSET
-                        for (int j= 0; j < val; j++) {
+                        for (int j = 0; j < val; j++) {
-                          assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i
+                            assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES
+                                                               // + i
                         }
                         break;
                     default:
@@ -162,12 +155,9 @@ public abstract class RowsLogEvent extends LogEvent
         columnLen = (int) buffer.getPackedLong();
         columns = buffer.getBitmap(columnLen);
 
-        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT)
+        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) {
-        {
             changeColumns = buffer.getBitmap(columnLen);
-        }
+        } else {
-        else
-        {
             changeColumns = columns;
         }
 
@@ -176,46 +166,38 @@ public abstract class RowsLogEvent extends LogEvent
         rowsBuf = buffer.duplicate(dataSize);
     }
 
-    public final void fillTable(LogContext context)
+    public final void fillTable(LogContext context) {
-    {
         table = context.getTable(tableId);
 
         // end of statement check:
-        if ((flags & RowsLogEvent.STMT_END_F) != 0)
+        if ((flags & RowsLogEvent.STMT_END_F) != 0) {
-        {
             // Now is safe to clear ignored map (clear_tables will also
             // delete original table map events stored in the map).
             context.clearAllTables();
         }
     }
 
-    public final long getTableId()
+    public final long getTableId() {
-    {
         return tableId;
     }
 
-    public final TableMapLogEvent getTable()
+    public final TableMapLogEvent getTable() {
-    {
         return table;
     }
 
-    public final BitSet getColumns()
+    public final BitSet getColumns() {
-    {
         return columns;
     }
 
-    public final BitSet getChangeColumns()
+    public final BitSet getChangeColumns() {
-    {
         return changeColumns;
     }
 
-    public final RowsLogBuffer getRowsBuf(String charsetName)
+    public final RowsLogBuffer getRowsBuf(String charsetName) {
-    {
         return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName);
     }
 
-    public final int getFlags(final int flags)
+    public final int getFlags(final int flags) {
-    {
         return this.flags & flags;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java

@@ -5,21 +5,18 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and
- * 4.x).
+ * 4.x). Format_description_log_event derives from Start_log_event_v3; it is the
- * 
- * Format_description_log_event derives from Start_log_event_v3; it is the
  * Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that
  * describes the other events' Common-Header/Post-Header lengths. This event is
  * sent by MySQL 5.0 whenever it starts sending a new binlog if the requested
  * position is >4 (otherwise if ==4 the event will be sent naturally).
  * 
  * @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class StartLogEventV3 extends LogEvent
+public class StartLogEventV3 extends LogEvent {
-{
+
     /**
      * We could have used SERVER_VERSION_LENGTH, but this introduces an obscure
      * dependency - if somebody decided to change SERVER_VERSION_LENGTH this
@@ -34,9 +31,7 @@ public class StartLogEventV3 extends LogEvent
     protected int           binlogVersion;
     protected String        serverVersion;
 
-    public StartLogEventV3(LogHeader header, LogBuffer buffer,
+    public StartLogEventV3(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         buffer.position(descriptionEvent.commonHeaderLen);
@@ -44,18 +39,15 @@ public class StartLogEventV3 extends LogEvent
         serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
     }
 
-    public StartLogEventV3()
+    public StartLogEventV3(){
-    {
         super(new LogHeader(START_EVENT_V3));
     }
 
-    public final String getServerVersion()
+    public final String getServerVersion() {
-    {
         return serverVersion;
     }
 
-    public final int getBinlogVersion()
+    public final int getBinlogVersion() {
-    {
         return binlogVersion;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java

@@ -4,19 +4,15 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * Stop_log_event.
+ * Stop_log_event. The Post-Header and Body for this event type are empty; it
- * 
+ * only has the Common-Header.
- * The Post-Header and Body for this event type are empty; it only has the
- * Common-Header.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class StopLogEvent extends LogEvent
+public final class StopLogEvent extends LogEvent {
-{
+
-    public StopLogEvent(LogHeader header, LogBuffer buffer,
+    public StopLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent description_event){
-            FormatDescriptionLogEvent description_event)
-    {
         super(header);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java

@@ -8,44 +8,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * In row-based mode, every row operation event is preceded by a
  * Table_map_log_event which maps a table definition to a number. The table
- * definition consists of database name, table name, and column definitions.
+ * definition consists of database name, table name, and column definitions. The
- * 
+ * Post-Header has the following components:
- * The Post-Header has the following components:
- * 
  * <table>
  * <caption>Post-Header for Table_map_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>table_id</td>
  * <td>6 bytes unsigned integer</td>
  * <td>The number that identifies the table.</td>
  * </tr>
- * 
  * <tr>
  * <td>flags</td>
  * <td>2 byte bitfield</td>
  * <td>Reserved for future use; currently always 0.</td>
  * </tr>
- * 
  * </table>
- * 
  * The Body has the following components:
- * 
  * <table>
  * <caption>Body for Table_map_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>database_name</td>
  * <td>one byte string length, followed by null-terminated string</td>
@@ -55,20 +45,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * by a terminating 0 byte. (Note the redundancy in the representation of the
  * length.)</td>
  * </tr>
- * 
  * <tr>
  * <td>table_name</td>
  * <td>one byte string length, followed by null-terminated string</td>
  * <td>The name of the table, encoded the same way as the database name above.</td>
  * </tr>
- * 
  * <tr>
  * <td>column_count</td>
  * <td>packed_integer "Packed Integer"</td>
  * <td>The number of columns in the table, represented as a packed
  * variable-length integer.</td>
  * </tr>
- * 
  * <tr>
  * <td>column_type</td>
  * <td>List of column_count 1 byte enumeration values</td>
@@ -78,13 +65,11 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * listed in the table Table_table_map_log_event_column_types "below" (along
  * with description of the associated metadata field).</td>
  * </tr>
- * 
  * <tr>
  * <td>metadata_length</td>
  * <td>packed_integer "Packed Integer"</td>
  * <td>The length of the following metadata block</td>
  * </tr>
- * 
  * <tr>
  * <td>metadata</td>
  * <td>list of metadata for each column</td>
@@ -93,7 +78,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * metadata for each column are listed in the table
  * Table_table_map_log_event_column_types "below".</td>
  * </tr>
- * 
  * <tr>
  * <td>null_bits</td>
  * <td>column_count bits, rounded up to nearest byte</td>
@@ -104,13 +88,10 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * first byte, the ninth is in the least significant bit of the second byte, and
  * so on.</td>
  * </tr>
- * 
  * </table>
- * 
  * The table below lists all column types, along with the numerical identifier
  * for it and the size and interpretation of meta-data used to describe the
  * type.
- * 
  * <table>
  * <caption>Table_map_log_event column types: numerical identifier and
  * metadata</caption>
@@ -120,35 +101,30 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <th>Size of metadata in bytes</th>
  * <th>Description of metadata</th>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DECIMAL</td>
  * <td>0</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TINY</td>
  * <td>1</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_SHORT</td>
  * <td>2</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_LONG</td>
  * <td>3</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_FLOAT</td>
  * <td>4</td>
@@ -156,7 +132,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>1 byte unsigned integer, representing the "pack_length", which is equal
  * to sizeof(float) on the server from which the event originates.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DOUBLE</td>
  * <td>5</td>
@@ -164,63 +139,54 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>1 byte unsigned integer, representing the "pack_length", which is equal
  * to sizeof(double) on the server from which the event originates.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_NULL</td>
  * <td>6</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TIMESTAMP</td>
  * <td>7</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_LONGLONG</td>
  * <td>8</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_INT24</td>
  * <td>9</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DATE</td>
  * <td>10</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TIME</td>
  * <td>11</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_DATETIME</td>
  * <td>12</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_YEAR</td>
  * <td>13</td>
  * <td>0</td>
  * <td>No column metadata.</td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_NEWDATE</i></td>
  * <td><i>14</i></td>
@@ -228,14 +194,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_VARCHAR</td>
  * <td>15</td>
  * <td>2 bytes</td>
  * <td>2 byte unsigned integer representing the maximum length of the string.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_BIT</td>
  * <td>16</td>
@@ -245,7 +209,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * occupied by the bitfield. The number of bytes is either int((length+7)/8) or
  * int(length/8).</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_NEWDECIMAL</td>
  * <td>246</td>
@@ -253,7 +216,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>A 1 byte unsigned int representing the precision, followed by a 1 byte
  * unsigned int representing the number of decimals.</td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_ENUM</i></td>
  * <td><i>247</i></td>
@@ -261,7 +223,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_SET</i></td>
  * <td><i>248</i></td>
@@ -269,7 +230,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_TINY_BLOB</td>
  * <td>249</td>
@@ -277,7 +237,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_MEDIUM_BLOB</i></td>
  * <td><i>250</i></td>
@@ -285,7 +244,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td><i>MYSQL_TYPE_LONG_BLOB</i></td>
  * <td><i>251</i></td>
@@ -293,7 +251,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td><i>This enumeration value is only used internally and cannot exist in a
  * binlog.</i></td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_BLOB</td>
  * <td>252</td>
@@ -301,7 +258,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>The pack length, i.e., the number of bytes needed to represent the length
  * of the blob: 1, 2, 3, or 4.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_VAR_STRING</td>
  * <td>253</td>
@@ -312,7 +268,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * unsigned integer representing the field size, i.e., the number of bytes
  * needed to store the length of the string.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_STRING</td>
  * <td>254</td>
@@ -321,7 +276,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * byte is the field size, i.e., the number of bytes in the representation of
  * size of the string: 3 or 4.</td>
  * </tr>
- * 
  * <tr>
  * <td>MYSQL_TYPE_GEOMETRY</td>
  * <td>255</td>
@@ -329,21 +283,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * <td>The pack length, i.e., the number of bytes needed to represent the length
  * of the geometry: 1, 2, 3, or 4.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class TableMapLogEvent extends LogEvent
+public final class TableMapLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part:
      * <ul>
      * <li>6 bytes. The table ID.</li>
      * <li>2 bytes. Reserved for future use.</li>
      * </ul>
-     * 
      * <p>
      * Variable data part:
      * <ul>
@@ -360,7 +312,6 @@ public final class TableMapLogEvent extends LogEvent
      * one bit per column. For this field, the amount of storage required for N
      * columns is INT((N+7)/8) bytes.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     protected final String dbname;
@@ -369,14 +320,15 @@ public final class TableMapLogEvent extends LogEvent
     /**
      * Holding mysql column information.
      */
-    public static final class ColumnInfo
+    public static final class ColumnInfo {
-    {
+
         public int type;
         public int meta;
     }
 
     protected final int          columnCnt;
-    protected final ColumnInfo[] columnInfo;         // buffer for field metadata
+    protected final ColumnInfo[] columnInfo;         // buffer for field
+                                                      // metadata
 
     protected final long         tableId;
     protected BitSet             nullBits;
@@ -388,22 +340,20 @@ public final class TableMapLogEvent extends LogEvent
     /**
      * Constructor used by slave to read the event from the binary log.
      */
-    public TableMapLogEvent(LogHeader header, LogBuffer buffer,
+    public TableMapLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
         /* Read the post-header */
         buffer.position(commonHeaderLen + TM_MAPID_OFFSET);
-        if (postHeaderLen == 6)
+        if (postHeaderLen == 6) {
-        {
+            /*
-            /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
+             * Master is of an intermediate source tree before 5.1.4. Id is 4
+             * bytes
+             */
             tableId = buffer.getUint32();
-        }
+        } else {
-        else
-        {
             // DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN);
             tableId = buffer.getUlong48();
         }
@@ -419,15 +369,13 @@ public final class TableMapLogEvent extends LogEvent
         // Read column information from buffer
         columnCnt = (int) buffer.getPackedLong();
         columnInfo = new ColumnInfo[columnCnt];
-        for (int i = 0; i < columnCnt; i++)
+        for (int i = 0; i < columnCnt; i++) {
-        {
             ColumnInfo info = new ColumnInfo();
             info.type = buffer.getUint8();
             columnInfo[i] = info;
         }
 
-        if (buffer.position() < buffer.limit())
+        if (buffer.position() < buffer.limit()) {
-        {
             final int fieldSize = (int) buffer.getPackedLong();
             decodeFields(buffer, fieldSize);
             nullBits = buffer.getBitmap(columnCnt);
@@ -439,41 +387,38 @@ public final class TableMapLogEvent extends LogEvent
      * 
      * @see mysql-5.1.60/sql/rpl_utility.h
      */
-    private final void decodeFields(LogBuffer buffer, final int len)
+    private final void decodeFields(LogBuffer buffer, final int len) {
-    {
         final int limit = buffer.limit();
 
         buffer.limit(len + buffer.position());
-        for (int i = 0; i < columnCnt; i++)
+        for (int i = 0; i < columnCnt; i++) {
-        {
             ColumnInfo info = columnInfo[i];
 
-            switch (info.type)
+            switch (info.type) {
-            {
+                case MYSQL_TYPE_TINY_BLOB:
-            case MYSQL_TYPE_TINY_BLOB:
+                case MYSQL_TYPE_BLOB:
-            case MYSQL_TYPE_BLOB:
+                case MYSQL_TYPE_MEDIUM_BLOB:
-            case MYSQL_TYPE_MEDIUM_BLOB:
+                case MYSQL_TYPE_LONG_BLOB:
-            case MYSQL_TYPE_LONG_BLOB:
+                case MYSQL_TYPE_DOUBLE:
-            case MYSQL_TYPE_DOUBLE:
+                case MYSQL_TYPE_FLOAT:
-            case MYSQL_TYPE_FLOAT:
+                case MYSQL_TYPE_GEOMETRY:
-            case MYSQL_TYPE_GEOMETRY:
+                case MYSQL_TYPE_JSON:
-                /*
+                    /*
-                  These types store a single byte.
+                     * These types store a single byte.
-                */
+                     */
-                info.meta = buffer.getUint8();
+                    info.meta = buffer.getUint8();
-                break;
+                    break;
-            case MYSQL_TYPE_SET:
+                case MYSQL_TYPE_SET:
-            case MYSQL_TYPE_ENUM:
+                case MYSQL_TYPE_ENUM:
-                /*
+                    /*
-                 * log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This
+                     * log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This
-                 * enumeration value is only used internally and cannot
+                     * enumeration value is only used internally and cannot
-                 * exist in a binlog.
+                     * exist in a binlog.
-                 */
+                     */
-                logger.warn("This enumeration value is only used internally "
+                    logger.warn("This enumeration value is only used internally "
-                        + "and cannot exist in a binlog: type=" + info.type);
+                                + "and cannot exist in a binlog: type=" + info.type);
-                break;
+                    break;
-            case MYSQL_TYPE_STRING:
+                case MYSQL_TYPE_STRING: {
-                {
                     /*
                      * log_event.h : The first byte is always
                      * MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the
@@ -485,59 +430,52 @@ public final class TableMapLogEvent extends LogEvent
                     info.meta = x;
                     break;
                 }
-            case MYSQL_TYPE_BIT:
+                case MYSQL_TYPE_BIT:
-                info.meta = buffer.getUint16();
+                    info.meta = buffer.getUint16();
-                break;
+                    break;
-            case MYSQL_TYPE_VARCHAR:
+                case MYSQL_TYPE_VARCHAR:
-                /*
+                    /*
-                 * These types store two bytes.
+                     * These types store two bytes.
-                 */
+                     */
-                info.meta = buffer.getUint16();
+                    info.meta = buffer.getUint16();
-                break;
+                    break;
-            case MYSQL_TYPE_NEWDECIMAL:
+                case MYSQL_TYPE_NEWDECIMAL: {
-                {
                     int x = buffer.getUint8() << 8; // precision
                     x += buffer.getUint8(); // decimals
                     info.meta = x;
                     break;
                 }
-            case MYSQL_TYPE_TIME2:
+                case MYSQL_TYPE_TIME2:
-            case MYSQL_TYPE_DATETIME2:
+                case MYSQL_TYPE_DATETIME2:
-            case MYSQL_TYPE_TIMESTAMP2:
+                case MYSQL_TYPE_TIMESTAMP2: {
-                {
                     info.meta = buffer.getUint8();
                     break;
                 }
-            default:
+                default:
-                info.meta = 0;
+                    info.meta = 0;
-                break;
+                    break;
             }
         }
         buffer.limit(limit);
     }
 
-    public final String getDbName()
+    public final String getDbName() {
-    {
         return dbname;
     }
 
-    public final String getTableName()
+    public final String getTableName() {
-    {
         return tblname;
     }
 
-    public final int getColumnCnt()
+    public final int getColumnCnt() {
-    {
         return columnCnt;
     }
 
-    public final ColumnInfo[] getColumnInfo()
+    public final ColumnInfo[] getColumnInfo() {
-    {
         return columnInfo;
     }
 
-    public final long getTableId()
+    public final long getTableId() {
-    {
         return tableId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java

@@ -8,10 +8,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UnknownLogEvent extends LogEvent
+public final class UnknownLogEvent extends LogEvent {
-{
+
-    public UnknownLogEvent(LogHeader header)
+    public UnknownLogEvent(LogHeader header){
-    {
         super(header);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java

@@ -4,19 +4,16 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
 /**
  * Log row updates with a before image. The event contain several update rows
- * for a table. Note that each event contains only rows for one table.
+ * for a table. Note that each event contains only rows for one table. Also note
- * 
+ * that the row data consists of pairs of row data: one row for the old data and
- * Also note that the row data consists of pairs of row data: one row for the
+ * one row for the new data.
- * old data and one row for the new data.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UpdateRowsLogEvent extends RowsLogEvent
+public final class UpdateRowsLogEvent extends RowsLogEvent {
-{
+
-    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer,
+    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java

@@ -8,22 +8,19 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
- * User_var_log_event.
+ * User_var_log_event. Every time a query uses the value of a user variable, a
- * 
+ * User_var_log_event is written before the Query_log_event, to set the user
- * Every time a query uses the value of a user variable, a User_var_log_event is
+ * variable.
- * written before the Query_log_event, to set the user variable.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UserVarLogEvent extends LogEvent
+public final class UserVarLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>4 bytes. the size of the user variable name.</li>
      * <li>The user variable name.</li>
@@ -39,7 +36,6 @@ public final class UserVarLogEvent extends LogEvent
      * <li>Variable-sized. For a string variable, this is the string. For a
      * float or integer variable, this is its value in 8 bytes.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final String       name;
@@ -66,80 +62,65 @@ public final class UserVarLogEvent extends LogEvent
     public static final int    UV_NAME_LEN_SIZE       = 4;
     public static final int    UV_CHARSET_NUMBER_SIZE = 4;
 
-    public UserVarLogEvent(LogHeader header, LogBuffer buffer,
+    public UserVarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
-            FormatDescriptionLogEvent descriptionEvent) throws IOException
+                                                                                                          throws IOException{
-    {
         super(header);
 
         /* The Post-Header is empty. The Variable Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
-                + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
         final int nameLen = (int) buffer.getUint32();
         name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE
         isNull = (0 != buffer.getInt8());
 
-        if (isNull)
+        if (isNull) {
-        {
             type = STRING_RESULT;
             charsetNumber = 63; /* binary */
             value = null;
-        }
+        } else {
-        else
-        {
             type = buffer.getInt8(); // UV_VAL_IS_NULL
             charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE
-            final int valueLen = (int) buffer.getUint32(); // buf +  UV_CHARSET_NUMBER_SIZE
+            final int valueLen = (int) buffer.getUint32(); // buf +
+                                                           // UV_CHARSET_NUMBER_SIZE
             final int limit = buffer.limit(); /* for restore */
             buffer.limit(buffer.position() + valueLen);
 
             /* @see User_var_log_event::print */
-            switch (type)
+            switch (type) {
-            {
+                case REAL_RESULT:
-            case REAL_RESULT:
+                    value = Double.valueOf(buffer.getDouble64()); // float8get
-                value = Double.valueOf(buffer.getDouble64()); // float8get
+                    break;
-                break;
+                case INT_RESULT:
-            case INT_RESULT:
+                    if (valueLen == 8) value = Long.valueOf(buffer.getLong64()); // !uint8korr
-                if (valueLen == 8)
+                    else if (valueLen == 4) value = Long.valueOf(buffer.getUint32());
-                    value = Long.valueOf(buffer.getLong64()); // !uint8korr
+                    else throw new IOException("Error INT_RESULT length: " + valueLen);
-                else if (valueLen == 4)
+                    break;
-                    value = Long.valueOf(buffer.getUint32());
+                case DECIMAL_RESULT:
-                else
+                    final int precision = buffer.getInt8();
-                    throw new IOException("Error INT_RESULT length: "
+                    final int scale = buffer.getInt8();
-                            + valueLen);
+                    value = buffer.getDecimal(precision, scale); // bin2decimal
-                break;
+                    break;
-            case DECIMAL_RESULT:
+                case STRING_RESULT:
-                final int precision = buffer.getInt8();
+                    String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
-                final int scale = buffer.getInt8();
+                    value = buffer.getFixString(valueLen, charsetName);
-                value = buffer.getDecimal(precision, scale); // bin2decimal
+                    break;
-                break;
+                case ROW_RESULT:
-            case STRING_RESULT:
+                    // this seems to be banned in MySQL altogether
-                String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
+                    throw new IOException("ROW_RESULT is unsupported");
-                value = buffer.getFixString(valueLen, charsetName);
+                default:
-                break;
+                    value = null;
-            case ROW_RESULT:
+                    break;
-                // this seems to be banned in MySQL altogether
-                throw new IOException("ROW_RESULT is unsupported");
-            default:
-                value = null;
-                break;
             }
             buffer.limit(limit);
         }
     }
 
-    public final String getQuery()
+    public final String getQuery() {
-    {
+        if (value == null) {
-        if (value == null)
-        {
             return "SET @" + name + " := NULL";
-        }
+        } else if (type == STRING_RESULT) {
-        else if (type == STRING_RESULT)
-        {
             // TODO: do escaping !?
             return "SET @" + name + " := \'" + value + '\'';
-        }
+        } else {
-        else
-        {
             return "SET @" + name + " := " + String.valueOf(value);
         }
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class WriteRowsLogEvent extends RowsLogEvent
+public final class WriteRowsLogEvent extends RowsLogEvent {
-{
+
-    public WriteRowsLogEvent(LogHeader header, LogBuffer buffer,
+    public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java

@@ -10,23 +10,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class XidLogEvent extends LogEvent
+public final class XidLogEvent extends LogEvent {
-{
+
     private final long xid;
 
-    public XidLogEvent(LogHeader header, LogBuffer buffer,
+    public XidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         /* The Post-Header is empty. The Variable Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
-                + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
         xid = buffer.getLong64(); // !uint8korr
     }
 
-    public final long getXid()
+    public final long getXid() {
-    {
         return xid;
     }
 }

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java

@@ -10,8 +10,8 @@ import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
 import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 
 public class BaseLogFetcherTest {
@@ -20,36 +20,38 @@ public class BaseLogFetcherTest {
     protected Charset charset        = Charset.forName("utf-8");
 
     protected void parseQueryEvent(QueryLogEvent event) {
-        System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
+        System.out.println(String.format("================> binlog[%s:%s] , name[%s]",
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+            binlogFileName,
-                                         event.getCatalog()));
+            event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+            event.getCatalog()));
         System.out.println("sql : " + event.getQuery());
     }
 
     protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
-        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+            .getLogPos() - event.getHeader().getEventLen()));
         System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
     }
 
     protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
-        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+            .getLogPos() - event.getHeader().getEventLen()));
         System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
     }
 
     protected void parseXidEvent(XidLogEvent event) throws Exception {
-        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
-                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+            .getLogPos() - event.getHeader().getEventLen()));
         System.out.println("xid : " + event.getXid());
     }
 
-
     protected void parseRowsEvent(RowsLogEvent event) {
         try {
-            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
+            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
-                                             event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+                binlogFileName,
-                                             event.getTable().getDbName(), event.getTable().getTableName()));
+                event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+                event.getTable().getDbName(),
+                event.getTable().getTableName()));
             RowsLogBuffer buffer = event.getRowsBuf(charset.name());
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
@@ -90,6 +92,10 @@ public class BaseLogFetcherTest {
         final ColumnInfo[] columnInfo = map.getColumnInfo();
 
         for (int i = 0; i < columnCnt; i++) {
+            if (!cols.get(i)) {
+                continue;
+            }
+
             ColumnInfo info = columnInfo[i];
             buffer.nextValue(info.type, info.meta);
 

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java

@@ -5,8 +5,7 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.Statement;
 
-import junit.framework.Assert;
+import org.junit.Assert;
-
 import org.junit.Test;
 
 import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
@@ -25,12 +24,12 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
         DirectLogFetcher fecther = new DirectLogFetcher();
         try {
             Class.forName("com.mysql.jdbc.Driver");
-            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "xxxxx", "xxxxx");
+            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello");
             Statement statement = connection.createStatement();
             statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
             statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
 
-            fecther.open(connection, "mysql-bin.000003", 4L, 2);
+            fecther.open(connection, "mysql-bin.000001", 4L, 2);
 
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
@@ -39,7 +38,8 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
                 event = decoder.decode(fecther, context);
 
                 if (event == null) {
-                    throw new RuntimeException("parse failed");
+                    continue;
+                    // throw new RuntimeException("parse failed");
                 }
 
                 int eventType = event.getHeader().getType();

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java

@@ -4,8 +4,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URL;
 
-import junit.framework.Assert;
+import org.junit.Assert;
-
 import org.junit.Before;
 import org.junit.Test;
 
@@ -27,7 +26,6 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
         URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
         File dummyFile = new File(url.getFile());
         directory = new File(dummyFile.getParent() + "/binlog").getPath();
-        // directory = "/home/jianghang/tmp/binlog";
     }
 
     @Test
@@ -37,8 +35,8 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
 
-            File current = new File(directory, "mysql-bin.000006");
+            File current = new File(directory, "mysql-bin.000001");
-            fetcher.open(current);
+            fetcher.open(current, 2051L);
             context.setLogPosition(new LogPosition(current.getName()));
 
             while (fetcher.fetch()) {
@@ -48,23 +46,22 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
                     int eventType = event.getHeader().getType();
                     switch (eventType) {
                         case LogEvent.ROTATE_EVENT:
-                             binlogFileName = ((RotateLogEvent)
+                            binlogFileName = ((RotateLogEvent) event).getFilename();
-                             event).getFilename();
                             break;
                         case LogEvent.WRITE_ROWS_EVENT_V1:
                         case LogEvent.WRITE_ROWS_EVENT:
-                             parseRowsEvent((WriteRowsLogEvent) event);
+                            parseRowsEvent((WriteRowsLogEvent) event);
                             break;
                         case LogEvent.UPDATE_ROWS_EVENT_V1:
                         case LogEvent.UPDATE_ROWS_EVENT:
-                             parseRowsEvent((UpdateRowsLogEvent) event);
+                            parseRowsEvent((UpdateRowsLogEvent) event);
                             break;
                         case LogEvent.DELETE_ROWS_EVENT_V1:
                         case LogEvent.DELETE_ROWS_EVENT:
-                             parseRowsEvent((DeleteRowsLogEvent) event);
+                            parseRowsEvent((DeleteRowsLogEvent) event);
                             break;
                         case LogEvent.QUERY_EVENT:
-                             parseQueryEvent((QueryLogEvent) event);
+                            parseQueryEvent((QueryLogEvent) event);
                             break;
                         case LogEvent.ROWS_QUERY_LOG_EVENT:
                             parseRowsQueryEvent((RowsQueryLogEvent) event);

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java

@@ -5,14 +5,11 @@ import java.math.BigInteger;
 
 import junit.framework.TestCase;
 
-import com.taobao.tddl.dbsync.binlog.LogBuffer;
+public class LogBufferTest extends TestCase {
 
-public class LogBufferTest extends TestCase
-{
     public static final int LOOP = 10000;
 
-    public void testSigned()
+    public void testSigned() {
-    {
         byte[] array = { 0, 0, 0, (byte) 0xff };
 
         LogBuffer buffer = new LogBuffer(array, 0, array.length);
@@ -24,17 +21,15 @@ public class LogBufferTest extends TestCase
         System.out.println(buffer.getUint24(1));
     }
 
-    public void testBigInteger()
+    public void testBigInteger() {
-    {
+        byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
-        byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
+                (byte) 0xff };
-                (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
 
         LogBuffer buffer = new LogBuffer(array, 0, array.length);
 
         long tt1 = 0;
         long l1 = 0;
-        for (int i = 0; i < LOOP; i++)
+        for (int i = 0; i < LOOP; i++) {
-        {
             final long t1 = System.nanoTime();
             l1 = buffer.getLong64(0);
             tt1 += System.nanoTime() - t1;
@@ -45,8 +40,7 @@ public class LogBufferTest extends TestCase
 
         long tt2 = 0;
         BigInteger l2 = null;
-        for (int i = 0; i < LOOP; i++)
+        for (int i = 0; i < LOOP; i++) {
-        {
             final long t2 = System.nanoTime();
             l2 = buffer.getUlong64(0);
             tt2 += System.nanoTime() - t2;
@@ -57,18 +51,15 @@ public class LogBufferTest extends TestCase
     }
 
     /* Reads big-endian integer from no more than 4 bytes */
-    private static int convertNBytesToInt(byte[] buffer, int offset, int length)
+    private static int convertNBytesToInt(byte[] buffer, int offset, int length) {
-    {
         int ret = 0;
-        for (int i = offset; i < (offset + length); i++)
+        for (int i = offset; i < (offset + length); i++) {
-        {
             ret = (ret << 8) | (0xff & buffer[i]);
         }
         return ret;
     }
 
-    private static int convert4BytesToInt(byte[] buffer, int offset)
+    private static int convert4BytesToInt(byte[] buffer, int offset) {
-    {
         int value;
         value = (0xff & buffer[offset + 3]);
         value += (0xff & buffer[offset + 2]) << 8;
@@ -77,24 +68,20 @@ public class LogBufferTest extends TestCase
         return value;
     }
 
-    public static short convert1ByteToShort(byte[] buffer, int offset)
+    public static short convert1ByteToShort(byte[] buffer, int offset) {
-    {
         short value;
         value = (short) buffer[offset + 0];
         return value;
     }
 
-    public static short convert2bytesToShort(byte[] buffer, int offset)
+    public static short convert2bytesToShort(byte[] buffer, int offset) {
-    {
         short value;
         value = (short) (buffer[offset + 0] << 8);
         value += (short) (buffer[offset + 1] & 0xff);
         return value;
     }
 
-    public static final BigDecimal extractDecimal(byte[] buffer, int precision,
+    public static final BigDecimal extractDecimal(byte[] buffer, int precision, int scale) {
-            int scale)
-    {
         //
         // Decimal representation in binlog seems to be as follows:
         // 1 byte - 'precision'
@@ -129,34 +116,28 @@ public class LogBufferTest extends TestCase
         int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
 
         // how many bytes are used to represent given amount of digits?
-        int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
+        int integerSize = intg0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[intg0x];
-                + LogBuffer.dig2bytes[intg0x];
+        int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[frac0x];
-        int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
-                + LogBuffer.dig2bytes[frac0x];
 
         int bin_size = integerSize + decimalSize; // total bytes
         byte[] d_copy = new byte[bin_size];
 
-        if (bin_size > buffer.length)
+        if (bin_size > buffer.length) {
-        {
+            throw new ArrayIndexOutOfBoundsException("Calculated bin_size: " + bin_size + ", available bytes: "
-            throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
+                                                     + buffer.length);
-                    + bin_size + ", available bytes: " + buffer.length);
         }
 
         // Invert first bit
         d_copy[0] = buffer[0];
         d_copy[0] ^= 0x80;
-        if (sign == -1)
+        if (sign == -1) {
-        {
             // Invert every byte
             d_copy[0] ^= 0xFF;
         }
 
-        for (int i = 1; i < bin_size; i++)
+        for (int i = 1; i < bin_size; i++) {
-        {
             d_copy[i] = buffer[i];
-            if (sign == -1)
+            if (sign == -1) {
-            {
                 // Invert every byte
                 d_copy[i] ^= 0xFF;
             }
@@ -167,62 +148,49 @@ public class LogBufferTest extends TestCase
 
         BigDecimal intPart = new BigDecimal(0);
 
-        if (offset > 0)
+        if (offset > 0) intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
-            intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
 
-        while (offset < integerSize)
+        while (offset < integerSize) {
-        {
+            intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(BigDecimal.valueOf(convert4BytesToInt(d_copy,
-            intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
+                offset)));
-                    BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
             offset += 4;
         }
 
         // Decimal part
         BigDecimal fracPart = new BigDecimal(0);
         int shift = 0;
-        for (int i = 0; i < frac0; i++)
+        for (int i = 0; i < frac0; i++) {
-        {
             shift += LogBuffer.DIG_PER_DEC1;
-            fracPart = fracPart.add(BigDecimal.valueOf(
+            fracPart = fracPart.add(BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
-                    convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
             offset += 4;
         }
 
-        if (LogBuffer.dig2bytes[frac0x] > 0)
+        if (LogBuffer.dig2bytes[frac0x] > 0) {
-        {
+            fracPart = fracPart.add(BigDecimal.valueOf(convertNBytesToInt(d_copy, offset, LogBuffer.dig2bytes[frac0x]))
-            fracPart = fracPart.add(BigDecimal.valueOf(
+                .movePointLeft(shift + frac0x));
-                    convertNBytesToInt(d_copy, offset,
-                            LogBuffer.dig2bytes[frac0x])).movePointLeft(
-                    shift + frac0x));
         }
 
         return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
     }
 
-    public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
+    public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b, 0x38, (byte) 0xb0, 0x60, 0x00 };
-            0x38, (byte) 0xb0, 0x60, 0x00 };
 
-    public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
+    public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7,
-            (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
+            (byte) 0x4f, (byte) 0xa0, (byte) 0xff };
-            (byte) 0xa0, (byte) 0xff };
 
     public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
 
     public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
 
-    public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
+    public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96, -108 };
-            -108                     };
 
-    public void testBigDecimal() throws InterruptedException
+    public void testBigDecimal() throws InterruptedException {
-    {
+        do {
-        do
-        {
             System.out.println("old extract decimal: ");
 
             long tt1 = 0;
             BigDecimal bd1 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t1 = System.nanoTime();
                 bd1 = extractDecimal(array2, 19, 10);
                 tt1 += System.nanoTime() - t1;
@@ -233,8 +201,7 @@ public class LogBufferTest extends TestCase
 
             long tt2 = 0;
             BigDecimal bd2 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t2 = System.nanoTime();
                 bd2 = extractDecimal(array1, 19, 10);
                 tt2 += System.nanoTime() - t2;
@@ -245,8 +212,7 @@ public class LogBufferTest extends TestCase
 
             long tt3 = 0;
             BigDecimal bd3 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t3 = System.nanoTime();
                 bd3 = extractDecimal(array3, 18, 6);
                 tt3 += System.nanoTime() - t3;
@@ -257,8 +223,7 @@ public class LogBufferTest extends TestCase
 
             long tt4 = 0;
             BigDecimal bd4 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t4 = System.nanoTime();
                 bd4 = extractDecimal(array4, 18, 6);
                 tt4 += System.nanoTime() - t4;
@@ -269,8 +234,7 @@ public class LogBufferTest extends TestCase
 
             long tt5 = 0;
             BigDecimal bd5 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t5 = System.nanoTime();
                 bd5 = extractDecimal(array5, 18, 6);
                 tt5 += System.nanoTime() - t5;
@@ -278,11 +242,9 @@ public class LogBufferTest extends TestCase
             System.out.print(tt5 / LOOP);
             System.out.print("ns >> ");
             System.out.println(bd5);
-        }
+        } while (false);
-        while (false);
 
-        do
+        do {
-        {
             System.out.println("new extract decimal: ");
 
             LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
@@ -293,8 +255,7 @@ public class LogBufferTest extends TestCase
 
             long tt1 = 0;
             BigDecimal bd1 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t1 = System.nanoTime();
                 bd1 = buffer1.getDecimal(0, 19, 10);
                 tt1 += System.nanoTime() - t1;
@@ -305,8 +266,7 @@ public class LogBufferTest extends TestCase
 
             long tt2 = 0;
             BigDecimal bd2 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t2 = System.nanoTime();
                 bd2 = buffer2.getDecimal(0, 19, 10);
                 tt2 += System.nanoTime() - t2;
@@ -317,8 +277,7 @@ public class LogBufferTest extends TestCase
 
             long tt3 = 0;
             BigDecimal bd3 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t3 = System.nanoTime();
                 bd3 = buffer3.getDecimal(0, 18, 6);
                 tt3 += System.nanoTime() - t3;
@@ -329,8 +288,7 @@ public class LogBufferTest extends TestCase
 
             long tt4 = 0;
             BigDecimal bd4 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t4 = System.nanoTime();
                 bd4 = buffer4.getDecimal(0, 18, 6);
                 tt4 += System.nanoTime() - t4;
@@ -341,8 +299,7 @@ public class LogBufferTest extends TestCase
 
             long tt5 = 0;
             BigDecimal bd5 = null;
-            for (int i = 0; i < LOOP; i++)
+            for (int i = 0; i < LOOP; i++) {
-            {
                 final long t5 = System.nanoTime();
                 bd5 = buffer5.getDecimal(0, 18, 6);
                 tt5 += System.nanoTime() - t5;
@@ -350,7 +307,6 @@ public class LogBufferTest extends TestCase
             System.out.print(tt5 / LOOP);
             System.out.print("ns >> ");
             System.out.println(bd5);
-        }
+        } while (false);
-        while (false);
     }
 }

deployer/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

deployer/src/main/bin/stop.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 
 cygwin=false;
+linux=false;
 case "`uname`" in
     CYGWIN*)
         cygwin=true
         ;;
+    Linux*)
+    	linux=true
+    	;;
 esac
 
 get_pid() {	
@@ -15,11 +19,19 @@ get_pid() {
         JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
         JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
     else
-        if [ ! -z "$PID" ]; then
+    	if $linux; then
-        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+	        if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
 	    else
-	        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+	    	if [ ! -z "$PID" ]; then
-        fi
+	        	JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    fi
     fi
     echo $JAVA_PID;
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java

@@ -5,6 +5,7 @@ import java.util.Properties;
 
 import org.I0Itec.zkclient.IZkStateListener;
 import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zookeeper.Watcher.Event.KeeperState;
@@ -15,6 +16,7 @@ import org.springframework.beans.factory.BeanFactory;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
+import com.alibaba.otter.canal.common.CanalException;
 import com.alibaba.otter.canal.common.utils.AddressUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
@@ -32,11 +34,12 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
 import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
 import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
 import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
-import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
 import com.alibaba.otter.canal.server.exception.CanalServerException;
 import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * canal调度控制器
@@ -58,7 +61,7 @@ public class CanalController {
     private boolean                                  autoScan = true;
     private InstanceAction                           defaultAction;
     private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
-    private CanalServerWithEmbeded                   embededCanalServer;
+    private CanalServerWithEmbedded                  embededCanalServer;
     private CanalServerWithNetty                     canalServer;
 
     private CanalInstanceGenerator                   instanceGenerator;
@@ -69,7 +72,7 @@ public class CanalController {
     }
 
     public CanalController(final Properties properties){
-        managerClients = new MapMaker().makeComputingMap(new Function<String, CanalConfigClient>() {
+        managerClients = MigrateMap.makeComputingMap(new Function<String, CanalConfigClient>() {
 
             public CanalConfigClient apply(String managerAddress) {
                 return getManagerClient(managerAddress);
@@ -86,9 +89,9 @@ public class CanalController {
         cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
         ip = getProperty(properties, CanalConstants.CANAL_IP);
         port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT));
-        embededCanalServer = new CanalServerWithEmbeded();
+        embededCanalServer = CanalServerWithEmbedded.instance();
         embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
-        canalServer = new CanalServerWithNetty(embededCanalServer);
+        canalServer = CanalServerWithNetty.instance();
         canalServer.setIp(ip);
         canalServer.setPort(port);
 
@@ -106,7 +109,7 @@ public class CanalController {
 
         final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
         ServerRunningMonitors.setServerData(serverData);
-        ServerRunningMonitors.setRunningMonitors(new MapMaker().makeComputingMap(new Function<String, ServerRunningMonitor>() {
+        ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
 
             public ServerRunningMonitor apply(final String destination) {
                 ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
@@ -146,6 +149,11 @@ public class CanalController {
                                     public void handleNewSession() throws Exception {
                                         initCid(path);
                                     }
+
+                                    @Override
+                                    public void handleSessionEstablishmentError(Throwable error) throws Exception {
+                                        logger.error("failed to connect to zookeeper", error);
+                                    }
                                 });
                             }
                         } finally {
@@ -170,6 +178,8 @@ public class CanalController {
                 if (zkclientx != null) {
                     runningMonitor.setZkClient(zkclientx);
                 }
+                // 触发创建一下cid节点
+                runningMonitor.init();
                 return runningMonitor;
             }
         }));
@@ -182,14 +192,15 @@ public class CanalController {
                 public void start(String destination) {
                     InstanceConfig config = instanceConfigs.get(destination);
                     if (config == null) {
-                        config = new InstanceConfig(globalInstanceConfig);
+                        // 重新读取一下instance config
+                        config = parseInstanceConfig(properties, destination);
                         instanceConfigs.put(destination, config);
                     }
 
-                    if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
+                    if (!embededCanalServer.isStart(destination)) {
                         // HA机制启动
                         ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
-                        if (!runningMonitor.isStart()) {
+                        if (!config.getLazy() && !runningMonitor.isStart()) {
                             runningMonitor.start();
                         }
                     }
@@ -214,7 +225,7 @@ public class CanalController {
                 }
             };
 
-            instanceConfigMonitors = new MapMaker().makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
+            instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
 
                 public InstanceConfigMonitor apply(InstanceMode mode) {
                     int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
@@ -228,7 +239,13 @@ public class CanalController {
                         if (StringUtils.isEmpty(rootDir)) {
                             rootDir = "../conf";
                         }
-                        monitor.setRootConf(rootDir);
+
+                        if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
+                            monitor.setRootConf(rootDir);
+                        } else {
+                            // eclipse debug模式
+                            monitor.setRootConf("src/main/resources/");
+                        }
                         return monitor;
                     } else if (mode.isManager()) {
                         return new ManagerInstanceConfigMonitor();
@@ -283,6 +300,9 @@ public class CanalController {
                             System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination);
                             instanceGenerator.setBeanFactory(getBeanFactory(config.getSpringXml()));
                             return instanceGenerator.generate(destination);
+                        } catch (Throwable e) {
+                            logger.error("generator instance failed.", e);
+                            throw new CanalException(e);
                         } finally {
                             System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, "");
                         }
@@ -368,6 +388,11 @@ public class CanalController {
                 public void handleNewSession() throws Exception {
                     initCid(path);
                 }
+
+                @Override
+                public void handleSessionEstablishmentError(Throwable error) throws Exception {
+                    logger.error("failed to connect to zookeeper", error);
+                }
             });
         }
         // 优先启动embeded服务
@@ -377,16 +402,16 @@ public class CanalController {
             final String destination = entry.getKey();
             InstanceConfig config = entry.getValue();
             // 创建destination的工作节点
-            if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
+            if (!embededCanalServer.isStart(destination)) {
                 // HA机制启动
                 ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
-                if (!runningMonitor.isStart()) {
+                if (!config.getLazy() && !runningMonitor.isStart()) {
                     runningMonitor.start();
                 }
             }
 
             if (autoScan) {
-                instanceConfigMonitors.get(config.getMode()).regeister(destination, defaultAction);
+                instanceConfigMonitors.get(config.getMode()).register(destination, defaultAction);
             }
         }
 
@@ -436,6 +461,9 @@ public class CanalController {
                 String parentDir = path.substring(0, path.lastIndexOf('/'));
                 zkclientx.createPersistent(parentDir, true);
                 zkclientx.createEphemeral(path);
+            } catch (ZkNodeExistsException e) {
+                // ignore
+                // 因为第一次启动时创建了cid,但在stop/start的时可能会关闭和新建,允许出现NodeExists问题s
             }
 
         }

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java

@@ -4,7 +4,6 @@ import java.io.FileInputStream;
 import java.util.Properties;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,6 +20,10 @@ public class CanalLauncher {
 
     public static void main(String[] args) throws Throwable {
         try {
+            logger.info("## set default uncaught exception handler");
+            setGlobalUncaughtExceptionHandler();
+
+            logger.info("## load canal configurations");
             String conf = System.getProperty("canal.conf", "classpath:canal.properties");
             Properties properties = new Properties();
             if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
@@ -41,8 +44,7 @@ public class CanalLauncher {
                         logger.info("## stop the canal server");
                         controller.stop();
                     } catch (Throwable e) {
-                        logger.warn("##something goes wrong when stopping canal Server:\n{}",
+                        logger.warn("##something goes wrong when stopping canal Server:", e);
-                                    ExceptionUtils.getFullStackTrace(e));
                     } finally {
                         logger.info("## canal server is down.");
                     }
@@ -50,9 +52,17 @@ public class CanalLauncher {
 
             });
         } catch (Throwable e) {
-            logger.error("## Something goes wrong when starting up the canal Server:\n{}",
+            logger.error("## Something goes wrong when starting up the canal Server:", e);
-                         ExceptionUtils.getFullStackTrace(e));
             System.exit(0);
         }
     }
+
+    private static void setGlobalUncaughtExceptionHandler() {
+        Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+            @Override
+            public void uncaughtException(Thread t, Throwable e) {
+                logger.error("UnCaughtException", e);
+            }
+        });
+    }
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java

@@ -11,15 +11,15 @@ public interface InstanceAction {
     /**
      * 启动destination
      */
-    public void start(String destination);
+    void start(String destination);
 
     /**
      * 停止destination
      */
-    public void stop(String destination);
+    void stop(String destination);
 
     /**
      * 重载destination，可能需要stop,start操作，或者只是更新下内存配置
      */
-    public void reload(String destination);
+    void reload(String destination);
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java

@@ -10,7 +10,7 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
  */
 public interface InstanceConfigMonitor extends CanalLifeCycle {
 
-    public void regeister(String destination, InstanceAction action);
+    void register(String destination, InstanceAction action);
 
-    public void unRegeister(String destination);
+    void unregister(String destination);
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java

@@ -9,11 +9,11 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
  */
 public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle {
 
-    public void regeister(String destination, InstanceAction action) {
+    public void register(String destination, InstanceAction action) {
 
     }
 
-    public void unRegeister(String destination) {
+    public void unregister(String destination) {
 
     }
 

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java

@@ -13,7 +13,6 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -24,6 +23,7 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
 import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * 监听基于spring配置的instance变化
@@ -39,7 +39,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
     private long                             scanIntervalInSecond = 5;
     private InstanceAction                   defaultAction        = null;
     private Map<String, InstanceAction>      actions              = new MapMaker().makeMap();
-    private Map<String, InstanceConfigFiles> lastFiles            = new MapMaker().makeComputingMap(new Function<String, InstanceConfigFiles>() {
+    private Map<String, InstanceConfigFiles> lastFiles            = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>() {
 
                                                                       public InstanceConfigFiles apply(String destination) {
                                                                           return new InstanceConfigFiles(destination);
@@ -48,6 +48,8 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
     private ScheduledExecutorService         executor             = Executors.newScheduledThreadPool(1,
                                                                       new NamedThreadFactory("canal-instance-scan"));
 
+    private volatile boolean                 isFirst              = true;
+
     public void start() {
         super.start();
         Assert.notNull(rootConf, "root conf dir is null!");
@@ -57,6 +59,9 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
             public void run() {
                 try {
                     scan();
+                    if (isFirst) {
+                        isFirst = false;
+                    }
                 } catch (Throwable e) {
                     logger.error("scan failed", e);
                 }
@@ -72,7 +77,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         lastFiles.clear();
     }
 
-    public void regeister(String destination, InstanceAction action) {
+    public void register(String destination, InstanceAction action) {
         if (action != null) {
             actions.put(destination, action);
         } else {
@@ -80,7 +85,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         }
     }
 
-    public void unRegeister(String destination) {
+    public void unregister(String destination) {
         actions.remove(destination);
     }
 
@@ -121,13 +126,18 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
 
             if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
                 // 存在合法的instance.properties，并且第一次添加时，进行启动操作
-                notifyStart(instanceDir, destination);
+                notifyStart(instanceDir, destination, instanceConfigs);
             } else if (actions.containsKey(destination)) {
                 // 历史已经启动过
                 if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties
                     notifyStop(destination);
                 } else {
                     InstanceConfigFiles lastFile = lastFiles.get(destination);
+                    // 历史启动过 所以配置文件信息必然存在
+                    if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
+                        logger.error("[{}] is started, but not found instance file info.", destination);
+                    }
+
                     boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles());
                     // 通知变化
                     if (hasChanged) {
@@ -160,13 +170,22 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         }
     }
 
-    private void notifyStart(File instanceDir, String destination) {
+    private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) {
         try {
             defaultAction.start(destination);
             actions.put(destination, defaultAction);
+
+            // 启动成功后记录配置文件信息
+            InstanceConfigFiles lastFile = lastFiles.get(destination);
+            List<FileInfo> newFileInfo = new ArrayList<FileInfo>();
+            for (File instanceConfig : instanceConfigs) {
+                newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
+            }
+            lastFile.setInstanceFiles(newFileInfo);
+
             logger.info("auto notify start {} successful.", destination);
         } catch (Throwable e) {
-            logger.error("scan add found[{}] but start failed", destination, ExceptionUtils.getFullStackTrace(e));
+            logger.error(String.format("scan add found[%s] but start failed", destination), e);
         }
     }
 
@@ -174,9 +193,10 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         InstanceAction action = actions.remove(destination);
         try {
             action.stop(destination);
+            lastFiles.remove(destination);
             logger.info("auto notify stop {} successful.", destination);
         } catch (Throwable e) {
-            logger.error("scan delete found[{}] but stop failed", destination, ExceptionUtils.getFullStackTrace(e));
+            logger.error(String.format("scan delete found[%s] but stop failed", destination), e);
             actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除
         }
     }
@@ -188,9 +208,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
                 action.reload(destination);
                 logger.info("auto notify reload {} successful.", destination);
             } catch (Throwable e) {
-                logger.error("scan reload found[{}] but reload failed",
+                logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
-                    destination,
-                    ExceptionUtils.getFullStackTrace(e));
             }
         }
     }
@@ -293,5 +311,4 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
         }
 
     }
-
 }

deployer/src/main/resources/canal.properties

@@ -36,10 +36,16 @@ canal.instance.network.sendBufferSize = 16384
 canal.instance.network.soTimeout = 30
 
 # binlog filter config
+canal.instance.filter.druid.ddl = true
 canal.instance.filter.query.dcl = false
 canal.instance.filter.query.dml = false
 canal.instance.filter.query.ddl = false
 canal.instance.filter.table.error = false
+canal.instance.filter.rows = false
+
+# binlog format/image check
+canal.instance.binlog.format = ROW,STATEMENT,MIXED 
+canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
 
 # binlog ddl isolation
 canal.instance.get.ddl.isolation = false
@@ -54,9 +60,13 @@ canal.conf.dir = ../conf
 canal.auto.scan = true
 canal.auto.scan.interval = 5
 
+canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml
+#canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml
+
 canal.instance.global.mode = spring 
 canal.instance.global.lazy = false
 #canal.instance.global.manager.address = 127.0.0.1:1099
+#canal.instance.global.spring.xml = classpath:spring/local-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
 canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml

deployer/src/main/resources/example/instance.properties

@@ -1,27 +1,33 @@
 #################################################
 ## mysql serverId
-canal.instance.mysql.slaveId = 1234
+canal.instance.mysql.slaveId=0
-
 # position info
-canal.instance.master.address = 127.0.0.1:3306
+canal.instance.master.address=127.0.0.1:3306
-canal.instance.master.journal.name = 
+canal.instance.master.journal.name=
-canal.instance.master.position = 
+canal.instance.master.position=
-canal.instance.master.timestamp = 
+canal.instance.master.timestamp=
+
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+canal.instance.tsdb.dbUsername=canal
+canal.instance.tsdb.dbPassword=canal
+
 
 #canal.instance.standby.address =
 #canal.instance.standby.journal.name =
 #canal.instance.standby.position = 
 #canal.instance.standby.timestamp = 
-
 # username/password
-canal.instance.dbUsername = canal
+canal.instance.dbUsername=canal
-canal.instance.dbPassword = canal
+canal.instance.dbPassword=canal
-canal.instance.defaultDatabaseName =
+canal.instance.defaultDatabaseName=test
-canal.instance.connectionCharset = UTF-8
+canal.instance.connectionCharset=UTF-8
-
 # table regex
-canal.instance.filter.regex = .*\\..*
+canal.instance.filter.regex=.*\\..*
 # table black regex
-canal.instance.filter.black.regex =  
+canal.instance.filter.black.regex=
-
 #################################################

deployer/src/main/resources/example/rds_instance.properties

@@ -0,0 +1,32 @@
+#################################################
+# rds openapi binlog
+canal.instance.rds.open.url=https://rds.aliyuncs.com/
+canal.instance.rds.open.accesskey=
+canal.instance.rds.open.secretkey=
+canal.instance.rds.instanceId=
+canal.instance.rds.startTime=
+canal.instance.rds.endTime=
+
+# local binlog dir
+canal.instance.parser.directory=${canal.file.data.dir:../conf}/${canal.instance.destination:}/binlog
+# position info
+canal.instance.master.address=127.0.0.1:3306
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+#canal.instance.tsdb.dbUsername=canal
+#canal.instance.tsdb.dbPassword=canal
+
+# username/password
+canal.instance.dbUsername=canal
+canal.instance.dbPassword=canal
+canal.instance.defaultDatabaseName=test
+canal.instance.connectionCharset=UTF-8
+# table regex
+canal.instance.filter.regex=test\\..*
+# table black regex
+canal.instance.filter.black.regex=
+#################################################

deployer/src/main/resources/logback.xml

@@ -75,9 +75,7 @@
     </logger>
     
 	<root level="WARN">
-		<!--  
 		<appender-ref ref="STDOUT"/>
-		-->
 		<appender-ref ref="CANAL-ROOT" />
 	</root>
 </configuration>

deployer/src/main/resources/spring/default-instance.xml

@@ -85,7 +85,7 @@
 
 	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -128,14 +128,14 @@
 		<!-- 解析位点记录 -->
 		<property name="logPositionManager">
 			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
-				<property name="primary">
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
-				</property>
+				</constructor-arg>
-				<property name="failback">
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
-						<property name="metaManager" ref="metaManager" />
+						<constructor-arg ref="metaManager"/>
 					</bean>
-				</property>
+				</constructor-arg>
 			</bean>
 		</property>
 		
@@ -178,6 +178,14 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
 	</bean>
 </beans>

deployer/src/main/resources/spring/file-instance.xml

@@ -71,7 +71,7 @@
 
 	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -113,14 +113,14 @@
 		<!-- 解析位点记录 -->
 		<property name="logPositionManager">
 			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
-				<property name="primary">
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
-				</property>
+				</constructor-arg>
-				<property name="failback">
+				<constructor-arg>
 					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
-						<property name="metaManager" ref="metaManager" />
+						<constructor-arg ref="metaManager"/>
 					</bean>
-				</property>
+				</constructor-arg>
 			</bean>
 		</property>
 		
@@ -163,6 +163,14 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
+
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
 	</bean>
 </beans>

deployer/src/main/resources/spring/group-instance.xml

@@ -77,7 +77,7 @@
 
 	<bean id="eventParser1" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -160,12 +160,15 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
 	</bean>
 	
 	<bean id="eventParser2" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -248,6 +251,10 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
 	</bean>
 </beans>

deployer/src/main/resources/spring/local-instance.xml

@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
+		<property name="destination" value="${canal.instance.destination}" />
+		<property name="eventParser">
+			<ref local="eventParser" />
+		</property>
+		<property name="eventSink">
+			<ref local="eventSink" />
+		</property>
+		<property name="eventStore">
+			<ref local="eventStore" />
+		</property>
+		<property name="metaManager">
+			<ref local="metaManager" />
+		</property>
+		<property name="alarmHandler">
+			<ref local="alarmHandler" />
+		</property>
+	</bean>
+	
+	<!-- 报警处理类 -->
+	<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
+	
+	<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
+		<property name="dataDir" value="${canal.file.data.dir:../conf}" />
+		<property name="period" value="${canal.file.flush.period:1000}" />
+	</bean>
+	
+	<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
+		<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
+		<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
+		<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
+		<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
+	</bean>
+	
+	<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
+		<property name="eventStore" ref="eventStore" />
+	</bean>
+
+	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser">
+		<property name="destination" value="${canal.instance.destination}" />			
+		<property name="alarmHandler" ref="alarmHandler" />
+
+		<!-- 解析过滤处理 -->
+		<property name="eventFilter">
+			<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
+				<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
+			</bean>
+		</property>
+		
+		<property name="eventBlackFilter">
+			<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
+				<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
+				<constructor-arg index="1" value="false" />
+			</bean>
+		</property>
+		
+		<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
+		<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
+			
+		<!-- 解析编码 -->
+		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
+	
+		<!-- 解析位点记录 -->
+		<property name="logPositionManager">
+			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
+				<constructor-arg>
+					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
+				</constructor-arg>
+				<constructor-arg>
+					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
+						<constructor-arg ref="metaManager"/>
+					</bean>
+				</constructor-arg>
+			</bean>
+		</property>
+		
+		<!-- 解析数据库信息 -->
+		<property name="masterInfo">
+			<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
+				<property name="address" value="${canal.instance.master.address}" />
+				<property name="username" value="${canal.instance.dbUsername:retl}" />
+				<property name="password" value="${canal.instance.dbPassword:retl}" />
+				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
+			</bean>
+		</property>
+		
+		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
+		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
+		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
+		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="needWait" value="${canal.instance.parser.needWait:false}"/>
+		<property name="directory" value="${canal.instance.parser.directory:}"/>
+		
+		<!-- rds相关 -->
+		<property name="url" value="${canal.instance.rds.open.url:}"/>
+		<property name="accesskey" value="${canal.instance.rds.open.accesskey:}"/>
+		<property name="secretkey" value="${canal.instance.rds.open.secretkey:}"/>
+		<property name="instanceId" value="${canal.instance.rds.instanceId:}"/>
+		<property name="startTime" value="${canal.instance.rds.startTime:}"/>
+		<property name="endTime" value="${canal.instance.rds.endTime:}"/>
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
+	</bean>
+</beans>

deployer/src/main/resources/spring/memory-instance.xml

@@ -68,7 +68,7 @@
 
 	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
-		<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
+		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
 		<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
 		<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -151,6 +151,14 @@
 		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
 		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
 		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
 		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="supportBinlogFormats" value="${canal.instance.binlog.format}" />
+		<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
 	</bean>
 </beans>

deployer/src/main/resources/spring/tsdb/h2-tsdb.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+	
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<!-- 基于db的实现 -->
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
+		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
+	</bean>
+	
+    <bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
+        <property name="driverClassName" value="org.h2.Driver" />
+		<property name="url" value="${canal.instance.tsdb.url:}" />
+		<property name="username" value="${canal.instance.tsdb.dbUsername:canal}" />
+		<property name="password" value="${canal.instance.tsdb.dbPassword:canal}" />
+      	<property name="maxActive" value="30" />
+        <property name="initialSize" value="0" />
+        <property name="minIdle" value="1" />
+        <property name="maxWait" value="10000" />
+        <property name="timeBetweenEvictionRunsMillis" value="60000" />
+        <property name="minEvictableIdleTimeMillis" value="300000" />
+        <property name="testWhileIdle" value="true" />
+        <property name="testOnBorrow" value="false" />
+        <property name="testOnReturn" value="false" />
+        <property name="useUnfairLock" value="true" />
+	</bean>
+
+    <bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
+        <property name="dataSource" ref="dataSource"/>
+        <property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
+    </bean>
+
+    <bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+
+    <bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+</beans>

deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+	
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<!-- 基于db的实现 -->
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
+		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
+	</bean>
+	
+    <bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
+        <property name="driverClassName" value="com.mysql.jdbc.Driver" />
+		<property name="url" value="${canal.instance.tsdb.url:}" />
+		<property name="username" value="${canal.instance.tsdb.dbUsername:canal}" />
+		<property name="password" value="${canal.instance.tsdb.dbPassword:canal}" />
+        <property name="maxActive" value="30" />
+        <property name="initialSize" value="0" />
+        <property name="minIdle" value="1" />
+        <property name="maxWait" value="10000" />
+        <property name="timeBetweenEvictionRunsMillis" value="60000" />
+        <property name="minEvictableIdleTimeMillis" value="300000" />
+        <property name="validationQuery" value="SELECT 1" />
+        <property name="exceptionSorterClassName" value="com.alibaba.druid.pool.vendor.MySqlExceptionSorter" />
+        <property name="validConnectionCheckerClassName" value="com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker" />
+        <property name="testWhileIdle" value="true" />
+        <property name="testOnBorrow" value="false" />
+        <property name="testOnReturn" value="false" />
+        <property name="useUnfairLock" value="true" />
+	</bean>
+
+    <bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
+        <property name="dataSource" ref="dataSource"/>
+        <property name="configLocation" value="classpath:spring/tsdb/sql-map/sqlmap-config.xml"/>
+    </bean>
+
+    <bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+
+    <bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
+        <property name="sqlMapClient" ref="sqlMapClient"/>
+    </bean>
+</beans>

deployer/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE sqlMapConfig PUBLIC "-//iBATIS.com//DTD SQL Map Config 2.0//EN"
+        "http://www.ibatis.com/dtd/sql-map-config-2.dtd">
+<sqlMapConfig>
+    <settings useStatementNamespaces="true"/>
+    <sqlMap resource="spring/tsdb/sql-map/sqlmap_history.xml"/>
+    <sqlMap resource="spring/tsdb/sql-map/sqlmap_snapshot.xml"/>
+</sqlMapConfig>

deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
+<sqlMap namespace="meta_history">
+    <typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO"/>
+    <sql id="allColumns">
+        <![CDATA[
+		gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,sql_schema,sql_table,sql_text,sql_type,extra
+        ]]>
+    </sql>
+    <sql id="allVOColumns">
+        <![CDATA[
+		a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
+		a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
+		a.use_schema as useSchema,a.sql_schema as sqlSchema,a.sql_table as sqlTable,a.sql_text as sqlText,a.sql_type as sqlType,a.extra as extra
+        ]]>
+    </sql>
+
+    <select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaHistoryDO">
+        select
+        <include refid="allVOColumns"/>
+        from meta_history a
+        <![CDATA[
+        where destination = #destination# and binlog_timestamp >= #snapshotTimestamp# and binlog_timestamp <= #timestamp#
+        order by binlog_timestamp asc,id asc
+        ]]>
+    </select>
+
+    <insert id="insert" parameterClass="metaHistoryDO">
+        insert into meta_history (<include refid="allColumns"/>)
+        values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#sqlSchema#,#sqlTable#,#sqlText#,#sqlType#,#extra#)
+    </insert>
+    
+    <delete id="deleteByName" parameterClass="java.util.Map">
+        delete from meta_history 
+        where destination=#destination#
+    </delete>
+
+
+    <delete id="deleteByGmtModified" parameterClass="java.util.Map">
+        <![CDATA[
+		delete from meta_history
+		where gmt_modified < timestamp(#timestamp#)
+        ]]>
+    </delete>
+</sqlMap>

deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
+<sqlMap namespace="meta_snapshot">
+    <typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO"/>
+    <typeAlias alias="tableMetaSnapshotDO"
+               type="com.alibaba.middleware.jingwei.biz.dataobject.CanalTableMetaSnapshotDO"/>
+    <sql id="allColumns">
+        <![CDATA[
+		gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
+        ]]>
+    </sql>
+    <sql id="allVOColumns">
+        <![CDATA[
+		a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
+		a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
+        ]]>
+    </sql>
+
+    <select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaSnapshotDO">
+    	select <include refid="allVOColumns"/>
+    	<![CDATA[
+        from meta_snapshot a
+        where destination = #destination# and binlog_timestamp < #timestamp#
+        order by binlog_timestamp desc,id desc
+        limit 1
+        ]]>
+    </select>
+    
+    <insert id="insert" parameterClass="metaSnapshotDO">
+        insert into meta_snapshot (<include refid="allColumns"/>)
+        values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#)
+    </insert>
+
+    <update id="update" parameterClass="metaSnapshotDO">
+        update meta_snapshot set gmt_modified=now(),
+        binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra#
+        where destination=#destination# and binlog_timestamp=0
+    </update>
+
+ 	<delete id="deleteByName" parameterClass="java.util.Map">
+        delete from meta_snapshot 
+        where destination=#destination#
+    </delete>
+
+    <delete id="deleteByGmtModified" parameterClass="java.util.Map">
+        <![CDATA[
+		delete from meta_snapshot
+		where gmt_modified < timestamp(#timestamp#)
+        ]]>
+    </delete>
+</sqlMap>

deployer/src/main/resources/spring/tsdb/sql/create_table.sql

@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS `meta_snapshot` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `gmt_create` datetime NOT NULL COMMENT '创建时间',
+  `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+  `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+  `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+  `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+  `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+  `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+  `data` longtext DEFAULT NULL COMMENT '表结构数据',
+  `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+  KEY `destination` (`destination`),
+  KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+  KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表';
+
+CREATE TABLE IF NOT EXISTS `meta_history` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
+  `gmt_create` datetime NOT NULL COMMENT '创建时间',
+  `gmt_modified` datetime NOT NULL COMMENT '修改时间',
+  `destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
+  `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
+  `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
+  `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
+  `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
+  `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
+  `schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
+  `table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
+  `sql` longtext DEFAULT NULL COMMENT '执行的sql',
+  `type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
+  `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
+  KEY `destination` (`destination`),
+  KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
+  KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表';

driver/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.0.25-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>
@@ -33,10 +33,6 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>org.jboss.netty</groupId>
-			<artifactId>netty</artifactId>
-		</dependency>
 		<!-- test dependency -->
 		<dependency>
 			<groupId>junit</groupId>

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java

@@ -2,19 +2,19 @@ package com.alibaba.otter.canal.parse.driver.mysql;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.SocketChannel;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.ClientAuthenticationPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QuitCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.HandshakeInitializationPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.Reply323Packet;
+import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
+import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannelPool;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
 
@@ -34,17 +34,20 @@ public class MysqlConnector {
     private byte                charsetNumber     = 33;
     private String              defaultSchema     = "retl";
     private int                 soTimeout         = 30 * 1000;
+    private int                 connTimeout       = 5 * 1000;
     private int                 receiveBufferSize = 16 * 1024;
     private int                 sendBufferSize    = 16 * 1024;
 
     private SocketChannel       channel;
+    private volatile boolean    dumping           = false;
+    // mysql connectinnId
+    private long                connectionId      = -1;
     private AtomicBoolean       connected         = new AtomicBoolean(false);
 
     public MysqlConnector(){
     }
 
     public MysqlConnector(InetSocketAddress address, String username, String password){
-
         this.address = address;
         this.username = username;
         this.password = password;
@@ -61,14 +64,12 @@ public class MysqlConnector {
     public void connect() throws IOException {
         if (connected.compareAndSet(false, true)) {
             try {
-                channel = SocketChannel.open();
+                channel = SocketChannelPool.open(address);
-                configChannel(channel);
                 logger.info("connect MysqlConnection to {}...", address);
-                channel.connect(address);
                 negotiate(channel);
             } catch (Exception e) {
                 disconnect();
-                throw new IOException("connect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e));
+                throw new IOException("connect " + this.address + " failure", e);
             }
         } else {
             logger.error("the channel can't be connected twice.");
@@ -86,10 +87,28 @@ public class MysqlConnector {
                 if (channel != null) {
                     channel.close();
                 }
-
                 logger.info("disConnect MysqlConnection to {}...", address);
             } catch (Exception e) {
-                throw new IOException("disconnect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e));
+                throw new IOException("disconnect " + this.address + " failure", e);
+            }
+
+            // 执行一次quit
+            if (dumping && connectionId >= 0) {
+                MysqlConnector connector = null;
+                try {
+                    connector = this.fork();
+                    connector.connect();
+                    MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector);
+                    executor.update("KILL CONNECTION " + connectionId);
+                } catch (Exception e) {
+                    throw new IOException("KILL DUMP " + connectionId + " failure", e);
+                } finally {
+                    if (connector != null) {
+                        connector.disconnect();
+                    }
+                }
+
+                dumping = false;
             }
         } else {
             logger.info("the channel {} is not connected", this.address);
@@ -110,18 +129,18 @@ public class MysqlConnector {
         connector.setReceiveBufferSize(getReceiveBufferSize());
         connector.setSendBufferSize(getSendBufferSize());
         connector.setSoTimeout(getSoTimeout());
+        connector.setConnTimeout(connTimeout);
         return connector;
     }
 
-    // ====================== help method ====================
+    public void quit() throws IOException {
+        QuitCommandPacket quit = new QuitCommandPacket();
+        byte[] cmdBody = quit.toBytes();
 
-    private void configChannel(SocketChannel channel) throws IOException {
+        HeaderPacket quitHeader = new HeaderPacket();
-        channel.socket().setKeepAlive(true);
+        quitHeader.setPacketBodyLength(cmdBody.length);
-        channel.socket().setReuseAddress(true);
+        quitHeader.setPacketSequenceNumber((byte) 0x00);
-        channel.socket().setSoTimeout(soTimeout);
+        PacketManager.writePkg(channel, quitHeader.toBytes(), cmdBody);
-        channel.socket().setTcpNoDelay(true);
-        channel.socket().setReceiveBufferSize(receiveBufferSize);
-        channel.socket().setSendBufferSize(sendBufferSize);
     }
 
     private void negotiate(SocketChannel channel) throws IOException {
@@ -140,6 +159,7 @@ public class MysqlConnector {
         }
         HandshakeInitializationPacket handshakePacket = new HandshakeInitializationPacket();
         handshakePacket.fromBytes(body);
+        connectionId = handshakePacket.threadId; // 记录一下connection
 
         logger.info("handshake initialization packet received, prepare the client authentication packet to send");
 
@@ -157,8 +177,7 @@ public class MysqlConnector {
         h.setPacketBodyLength(clientAuthPkgBody.length);
         h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
 
-        PacketManager.write(channel,
+        PacketManager.writePkg(channel, h.toBytes(), clientAuthPkgBody);
-            new ByteBuffer[] { ByteBuffer.wrap(h.toBytes()), ByteBuffer.wrap(clientAuthPkgBody) });
         logger.info("client authentication packet is sent out.");
 
         // check auth result
@@ -194,7 +213,7 @@ public class MysqlConnector {
         h323.setPacketBodyLength(b323Body.length);
         h323.setPacketSequenceNumber((byte) (packetSequenceNumber + 1));
 
-        PacketManager.write(channel, new ByteBuffer[] { ByteBuffer.wrap(h323.toBytes()), ByteBuffer.wrap(b323Body) });
+        PacketManager.writePkg(channel, h323.toBytes(), b323Body);
         logger.info("client 323 authentication packet is sent out.");
         // check auth result
         HeaderPacket header = PacketManager.readHeader(channel, 4);
@@ -291,4 +310,27 @@ public class MysqlConnector {
         this.password = password;
     }
 
+    public long getConnectionId() {
+        return connectionId;
+    }
+
+    public void setConnectionId(long connectionId) {
+        this.connectionId = connectionId;
+    }
+
+    public boolean isDumping() {
+        return dumping;
+    }
+
+    public void setDumping(boolean dumping) {
+        this.dumping = dumping;
+    }
+
+    public int getConnTimeout() {
+        return connTimeout;
+    }
+
+    public void setConnTimeout(int connTimeout) {
+        this.connTimeout = connTimeout;
+    }
 }

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java

@@ -1,17 +1,18 @@
 package com.alibaba.otter.canal.parse.driver.mysql;
 
 import java.io.IOException;
-import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
 import java.util.List;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.EOFPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetHeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.RowDataPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
 
 /**
@@ -24,9 +25,9 @@ public class MysqlQueryExecutor {
 
     private SocketChannel channel;
 
-    public MysqlQueryExecutor(MysqlConnector connector){
+    public MysqlQueryExecutor(MysqlConnector connector) throws IOException{
         if (!connector.isConnected()) {
-            throw new RuntimeException("should execute connector.connect() first");
+            throw new IOException("should execute connector.connect() first");
         }
 
         this.channel = connector.getChannel();
@@ -51,7 +52,7 @@ public class MysqlQueryExecutor {
         QueryCommandPacket cmd = new QueryCommandPacket();
         cmd.setQueryString(queryString);
         byte[] bodyBytes = cmd.toBytes();
-        PacketManager.write(channel, bodyBytes);
+        PacketManager.writeBody(channel, bodyBytes);
         byte[] body = readNextPacket();
 
         if (body[0] < 0) {
@@ -88,16 +89,70 @@ public class MysqlQueryExecutor {
         for (RowDataPacket r : rowData) {
             resultSet.getFieldValues().addAll(r.getColumns());
         }
-        resultSet.setSourceAddress(channel.socket().getRemoteSocketAddress());
+        resultSet.setSourceAddress(channel.getRemoteSocketAddress());
 
         return resultSet;
     }
 
-    private void readEofPacket() throws IOException {
+    public List<ResultSetPacket> queryMulti(String queryString) throws IOException {
+        QueryCommandPacket cmd = new QueryCommandPacket();
+        cmd.setQueryString(queryString);
+        byte[] bodyBytes = cmd.toBytes();
+        PacketManager.writeBody(channel, bodyBytes);
+        List<ResultSetPacket> resultSets = new ArrayList<ResultSetPacket>();
+        boolean moreResult = true;
+        while (moreResult) {
+            byte[] body = readNextPacket();
+            if (body[0] < 0) {
+                ErrorPacket packet = new ErrorPacket();
+                packet.fromBytes(body);
+                throw new IOException(packet + "\n with command: " + queryString);
+            }
+
+            ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
+            rsHeader.fromBytes(body);
+
+            List<FieldPacket> fields = new ArrayList<FieldPacket>();
+            for (int i = 0; i < rsHeader.getColumnCount(); i++) {
+                FieldPacket fp = new FieldPacket();
+                fp.fromBytes(readNextPacket());
+                fields.add(fp);
+            }
+
+            moreResult = readEofPacket();
+
+            List<RowDataPacket> rowData = new ArrayList<RowDataPacket>();
+            while (true) {
+                body = readNextPacket();
+                if (body[0] == -2) {
+                    break;
+                }
+                RowDataPacket rowDataPacket = new RowDataPacket();
+                rowDataPacket.fromBytes(body);
+                rowData.add(rowDataPacket);
+            }
+
+            ResultSetPacket resultSet = new ResultSetPacket();
+            resultSet.getFieldDescriptors().addAll(fields);
+            for (RowDataPacket r : rowData) {
+                resultSet.getFieldValues().addAll(r.getColumns());
+            }
+            resultSet.setSourceAddress(channel.getRemoteSocketAddress());
+            resultSets.add(resultSet);
+        }
+
+        return resultSets;
+    }
+
+    private boolean readEofPacket() throws IOException {
         byte[] eofBody = readNextPacket();
+        EOFPacket packet = new EOFPacket();
+        packet.fromBytes(eofBody);
         if (eofBody[0] != -2) {
             throw new IOException("EOF Packet is expected, but packet with field_count=" + eofBody[0] + " is found.");
         }
+
+        return (packet.statusFlag & 0x0008) != 0;
     }
 
     protected byte[] readNextPacket() throws IOException {

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java

@@ -1,7 +1,6 @@
 package com.alibaba.otter.canal.parse.driver.mysql;
 
 import java.io.IOException;
-import java.nio.channels.SocketChannel;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,28 +20,29 @@ public class MysqlUpdateExecutor {
 
     private static final Logger logger = LoggerFactory.getLogger(MysqlUpdateExecutor.class);
 
-    private SocketChannel       channel;
+    private MysqlConnector      connector;
 
-    public MysqlUpdateExecutor(MysqlConnector connector){
+    public MysqlUpdateExecutor(MysqlConnector connector) throws IOException{
         if (!connector.isConnected()) {
-            throw new RuntimeException("should execute connector.connect() first");
+            throw new IOException("should execute connector.connect() first");
         }
 
-        this.channel = connector.getChannel();
+        this.connector = connector;
     }
 
-    public MysqlUpdateExecutor(SocketChannel ch){
+    /*
-        this.channel = ch;
+     * public MysqlUpdateExecutor(SocketChannel ch){ this.channel = ch; }
-    }
+     */
 
     public OKPacket update(String updateString) throws IOException {
         QueryCommandPacket cmd = new QueryCommandPacket();
         cmd.setQueryString(updateString);
         byte[] bodyBytes = cmd.toBytes();
-        PacketManager.write(channel, bodyBytes);
+        PacketManager.writeBody(connector.getChannel(), bodyBytes);
 
         logger.debug("read update result...");
-        byte[] body = PacketManager.readBytes(channel, PacketManager.readHeader(channel, 4).getPacketBodyLength());
+        byte[] body = PacketManager.readBytes(connector.getChannel(),
+            PacketManager.readHeader(connector.getChannel(), 4).getPacketBodyLength());
         if (body[0] < 0) {
             ErrorPacket packet = new ErrorPacket();
             packet.fromBytes(body);

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java

@@ -14,15 +14,17 @@ import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
  * </pre>
  * 
  * <br>
- * The Packet Header will not be shown in the descriptions of packets that follow this section. Think of it as always
+ * The Packet Header will not be shown in the descriptions of packets that
- * there. But logically, it "precedes the packet" rather than "is included in the packet".<br>
+ * follow this section. Think of it as always there. But logically, it
+ * "precedes the packet" rather than "is included in the packet".<br>
  * 
  * @author fujohnwang
  */
 public class HeaderPacket implements IPacket {
 
     /**
-     * this field indicates the packet length that follows the header, with header packet's 4 bytes excluded.
+     * this field indicates the packet length that follows the header, with
+     * header packet's 4 bytes excluded.
      */
     private int  packetBodyLength;
     private byte packetSequenceNumber;

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java

@@ -12,6 +12,7 @@ import java.io.IOException;
  * @see 1.0
  */
 public interface IPacket {
+
     /**
      * un-marshal raw bytes into {@link IPacket} state for application usage.<br>
      * 

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java

@@ -47,10 +47,11 @@ public class ClientAuthenticationPacket extends PacketWithHeaderPacket {
         /**
          * CLIENT_LONG_PASSWORD CLIENT_LONG_FLAG CLIENT_PROTOCOL_41
          * CLIENT_INTERACTIVE CLIENT_TRANSACTIONS CLIENT_SECURE_CONNECTION
+         * CLIENT_MULTI_STATEMENTS;
          */
-        ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768, out); // remove
+        ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768 | 0x00010000, out); // remove
-                                                                                  // client_interactive
+        // client_interactive
-                                                                                  // feature
+        // feature
 
         // 2. write max_packet_size
         ByteHelper.writeUnsignedIntLittleEndian(MSC.MAX_PACKET_LENGTH, out);

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QuitCommandPacket.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
+
+/**
+ * quit cmd
+ * 
+ * @author agapple 2016年3月1日 下午8:33:02
+ * @since 1.0.22
+ */
+public class QuitCommandPacket extends CommandPacket {
+
+    public static final byte[] QUIT = new byte[] { 1, 0, 0, 0, 1 };
+
+    public QuitCommandPacket(){
+        setCommand((byte) 0x01);
+    }
+
+    @Override
+    public void fromBytes(byte[] data) throws IOException {
+
+    }
+
+    @Override
+    public byte[] toBytes() throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        out.write(getCommand());
+        out.write(QUIT);
+        return out.toByteArray();
+    }
+
+}

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java

@@ -74,6 +74,8 @@ public class FieldPacket extends PacketWithHeaderPacket {
         this.decimals = data[index];
         index++;
         //
+        index += 2;// skip filter
+        //
         if (index < data.length) {
             reader.setIndex(index);
             this.definition = reader.readLengthCodedString(data);