init

parse/src/main/java/com/alibaba/otter/canal/parse/CanalEventParser.java

@@ -0,0 +1,13 @@
+package com.alibaba.otter.canal.parse;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+
+/**
+ * 数据复制控制器
+ * 
+ * @author jianghang 2012-6-21 下午04:03:25
+ * @version 1.0.0
+ */
+public interface CanalEventParser<EVENT> extends CanalLifeCycle {
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/CanalHASwitchable.java

@@ -0,0 +1,16 @@
+package com.alibaba.otter.canal.parse;
+
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+
+/**
+ * 支持可切换的数据复制控制器
+ * 
+ * @author jianghang 2012-6-26 下午05:41:43
+ * @version 1.0.0
+ */
+public interface CanalHASwitchable {
+
+    public void doSwitch();
+
+    public void doSwitch(AuthenticationInfo newAuthenticationInfo);
+}

parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalHAException.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalHAException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalHAException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalHAException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalHAException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalHAException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalHAException(Throwable cause){
+        super(cause);
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalParseException.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalParseException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalParseException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalParseException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalParseException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalParseException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalParseException(Throwable cause){
+        super(cause);
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/exception/TableIdNotFoundException.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+public class TableIdNotFoundException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public TableIdNotFoundException(String errorCode){
+        super(errorCode);
+    }
+
+    public TableIdNotFoundException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public TableIdNotFoundException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public TableIdNotFoundException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public TableIdNotFoundException(Throwable cause){
+        super(cause);
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/ha/CanalHAController.java

@@ -0,0 +1,17 @@
+package com.alibaba.otter.canal.parse.ha;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalHAException;
+
+/**
+ * HA 控制器实现
+ * 
+ * @author jianghang 2012-6-26 下午05:21:07
+ * @version 1.0.0
+ */
+public interface CanalHAController extends CanalLifeCycle {
+
+    public void start() throws CanalHAException;
+
+    public void stop() throws CanalHAException;
+}

parse/src/main/java/com/alibaba/otter/canal/parse/ha/HeartBeatHAController.java

@@ -0,0 +1,63 @@
+package com.alibaba.otter.canal.parse.ha;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.CanalHASwitchable;
+import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback;
+
+/**
+ * 基于HeartBeat信息的HA控制 , 注意：非线程安全，需要做做多例化
+ * 
+ * @author jianghang 2012-7-6 下午02:33:30
+ * @version 1.0.0
+ */
+public class HeartBeatHAController extends AbstractCanalLifeCycle implements CanalHAController, HeartBeatCallback {
+
+    private static final Logger logger              = LoggerFactory.getLogger(HeartBeatHAController.class);
+    // default 3 times
+    private int                 detectingRetryTimes = 3;
+    private int                 failedTimes         = 0;
+    private boolean             switchEnable        = false;
+    private CanalHASwitchable   eventParser;
+
+    public HeartBeatHAController(){
+
+    }
+
+    public void onSuccess(long costTime) {
+        failedTimes = 0;
+    }
+
+    public void onFailed(Throwable e) {
+        failedTimes++;
+        // 检查一下是否超过失败次数
+        synchronized (this) {
+            if (failedTimes > detectingRetryTimes) {
+                if (switchEnable) {
+                    eventParser.doSwitch();// 通知执行一次切换
+                    failedTimes = 0;
+                } else {
+                    logger.warn("HeartBeat failed Times:{} , should auto switch ?", failedTimes);
+                }
+            }
+        }
+    }
+
+    // ============================= setter / getter
+    // ============================
+
+    public void setCanalHASwitchable(CanalHASwitchable canalHASwitchable) {
+        this.eventParser = canalHASwitchable;
+    }
+
+    public void setDetectingRetryTimes(int detectingRetryTimes) {
+        this.detectingRetryTimes = detectingRetryTimes;
+    }
+
+    public void setSwitchEnable(boolean switchEnable) {
+        this.switchEnable = switchEnable;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractBinlogParser.java

@@ -0,0 +1,25 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+
+public abstract class AbstractBinlogParser<T> extends AbstractCanalLifeCycle implements BinlogParser<T> {
+
+    public void reset() {
+    }
+
+    public Entry parse(T event, TableMeta tableMeta) throws CanalParseException {
+        return null;
+    }
+
+    public Entry parse(T event) throws CanalParseException {
+        return null;
+    }
+
+    public void stop() {
+        reset();
+        super.stop();
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java

@@ -0,0 +1,519 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang.math.RandomUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
+import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+/**
+ * 抽象的EventParser, 最大化共用mysql/oracle版本的实现
+ * 
+ * @author jianghang 2013-1-20 下午08:10:25
+ * @version 1.0.0
+ */
+public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle implements CanalEventParser<EVENT> {
+
+    protected final Logger                           logger                     = LoggerFactory.getLogger(this.getClass());
+
+    protected CanalLogPositionManager                logPositionManager         = null;
+    protected CanalEventSink<List<CanalEntry.Entry>> eventSink                  = null;
+    protected CanalEventFilter                       eventFilter                = null;
+    protected CanalEventFilter                       eventBlackFilter           = null;
+
+    private CanalAlarmHandler                        alarmHandler               = null;
+
+    // 统计参数
+    protected AtomicBoolean                          profilingEnabled           = new AtomicBoolean(false);                // profile开关参数
+    protected AtomicLong                             receivedEventCount         = new AtomicLong();
+    protected AtomicLong                             parsedEventCount           = new AtomicLong();
+    protected AtomicLong                             consumedEventCount         = new AtomicLong();
+    protected long                                   parsingInterval            = -1;
+    protected long                                   processingInterval         = -1;
+
+    // 认证信息
+    protected volatile AuthenticationInfo            runningInfo;
+    protected String                                 destination;
+
+    // binLogParser
+    protected BinlogParser                           binlogParser               = null;
+
+    protected Thread                                 parseThread                = null;
+
+    protected Thread.UncaughtExceptionHandler        handler                    = new Thread.UncaughtExceptionHandler() {
+
+                                                                                    public void uncaughtException(Thread t,
+                                                                                                                  Throwable e) {
+                                                                                        logger.error("parse events has an error",
+                                                                                            e);
+                                                                                    }
+                                                                                };
+
+    protected EventTransactionBuffer                 transactionBuffer;
+    protected int                                    transactionSize            = 1024;
+    protected AtomicBoolean                          needTransactionPosition    = new AtomicBoolean(false);
+    protected long                                   lastEntryTime              = 0L;
+    protected volatile boolean                       detectingEnable            = true;                                    // 是否开启心跳检查
+    protected Integer                                detectingIntervalInSeconds = 3;                                       // 检测频率
+    protected volatile Timer                         timer;
+    protected TimerTask                              heartBeatTimerTask;
+    protected Throwable                              exception                  = null;
+
+    protected abstract BinlogParser buildParser();
+
+    protected abstract ErosaConnection buildErosaConnection();
+
+    protected abstract EntryPosition findStartPosition(ErosaConnection connection) throws IOException;
+
+    protected void preDump(ErosaConnection connection) {
+    }
+
+    protected void afterDump(ErosaConnection connection) {
+    }
+
+    public void sendAlarm(String destination, String msg) {
+        if (this.alarmHandler != null) {
+            this.alarmHandler.sendAlarm(destination, msg);
+        }
+    }
+
+    public AbstractEventParser(){
+        // 初始化一下
+        transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {
+
+            public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
+                boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
+                if (!running) {
+                    return;
+                }
+
+                if (!successed) {
+                    throw new CanalParseException("consume failed!");
+                }
+
+                LogPosition position = buildLastTranasctionPosition(transaction);
+                if (position != null) { // 可能position为空
+                    logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
+                }
+            }
+        });
+    }
+
+    public void start() {
+        super.start();
+        MDC.put("destination", destination);
+        // 配置transaction buffer
+        // 初始化缓冲队列
+        transactionBuffer.setBufferSize(transactionSize);// 设置buffer大小
+        transactionBuffer.start();
+        // 构造bin log parser
+        binlogParser = buildParser();// 初始化一下BinLogParser
+        binlogParser.start();
+        // 启动工作线程
+        parseThread = new Thread(new Runnable() {
+
+            public void run() {
+                MDC.put("destination", String.valueOf(destination));
+                ErosaConnection erosaConnection = null;
+                while (running) {
+                    try {
+
+                        // 开始执行replication
+                        // 1. 构造Erosa连接
+                        erosaConnection = buildErosaConnection();
+
+                        // 2. 启动一个心跳线程
+                        startHeartBeat(erosaConnection);
+
+                        // 3. 执行dump前的准备工作
+                        preDump(erosaConnection);
+
+                        erosaConnection.connect();// 链接
+                        // 4. 获取最后的位置信息
+                        final EntryPosition startPosition = findStartPosition(erosaConnection);
+                        if (startPosition == null) {
+                            throw new CanalParseException("can't find start position for " + destination);
+                        }
+                        logger.info("find start position : {}", startPosition.toString());
+                        // 重新链接，因为在找position过程中可能有状态，需要断开后重建
+                        erosaConnection.reconnect();
+
+                        final SinkFunction sinkHandler = new SinkFunction<EVENT>() {
+
+                            private LogPosition lastPosition;
+
+                            public boolean sink(EVENT event) {
+                                try {
+                                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+
+                                    if (!running) {
+                                        return false;
+                                    }
+
+                                    if (entry != null) {
+                                        exception = null; // 有正常数据流过，清空exception
+                                        transactionBuffer.add(entry);
+                                        // 记录一下对应的positions
+                                        this.lastPosition = buildLastPosition(entry);
+                                        // 记录一下最后一次有数据的时间
+                                        lastEntryTime = System.currentTimeMillis();
+                                    }
+                                    return running;
+                                } catch (TableIdNotFoundException e) {
+                                    throw e;
+                                } catch (Exception e) {
+                                    // 记录一下，出错的位点信息
+                                    processError(e,
+                                        this.lastPosition,
+                                        startPosition.getJournalName(),
+                                        startPosition.getPosition());
+                                    throw new CanalParseException(e); // 继续抛出异常，让上层统一感知
+                                }
+                            }
+
+                        };
+
+                        // 4. 开始dump数据
+                        if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
+                            erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
+                        } else {
+                            erosaConnection.dump(startPosition.getJournalName(),
+                                startPosition.getPosition(),
+                                sinkHandler);
+                        }
+
+                    } catch (TableIdNotFoundException e) {
+                        exception = e;
+                        // 特殊处理TableIdNotFound异常,出现这样的异常，一种可能就是起始的position是一个事务当中，导致tablemap
+                        // Event时间没解析过
+                        needTransactionPosition.compareAndSet(false, true);
+                        logger.error(String.format("dump address %s has an error, retrying. caused by ",
+                            runningInfo.getAddress().toString()), e);
+                    } catch (Throwable e) {
+                        exception = e;
+                        if (!running) {
+                            if (!(e instanceof java.nio.channels.ClosedByInterruptException || e.getCause() instanceof java.nio.channels.ClosedByInterruptException)) {
+                                throw new CanalParseException(String.format("dump address %s has an error, retrying. ",
+                                    runningInfo.getAddress().toString()), e);
+                            }
+                        } else {
+                            logger.error(String.format("dump address %s has an error, retrying. caused by ",
+                                runningInfo.getAddress().toString()), e);
+                            sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
+                        }
+                    } finally {
+                        // 关闭一下链接
+                        afterDump(erosaConnection);
+                        try {
+                            if (erosaConnection != null) {
+                                erosaConnection.disconnect();
+                            }
+                        } catch (IOException e1) {
+                            if (!running) {
+                                throw new CanalParseException(String.format("disconnect address %s has an error, retrying. ",
+                                    runningInfo.getAddress().toString()),
+                                    e1);
+                            } else {
+                                logger.error("disconnect address {} has an error, retrying., caused by ",
+                                    runningInfo.getAddress().toString(),
+                                    e1);
+                            }
+                        }
+                    }
+                    // 出异常了，退出sink消费，释放一下状态
+                    eventSink.interrupt();
+                    transactionBuffer.reset();// 重置一下缓冲队列，重新记录数据
+                    binlogParser.reset();// 重新置位
+
+                    if (running) {
+                        // sleep一段时间再进行重试
+                        try {
+                            Thread.sleep(10000 + RandomUtils.nextInt(10000));
+                        } catch (InterruptedException e) {
+                        }
+                    }
+                }
+                MDC.remove("destination");
+            }
+        });
+
+        parseThread.setUncaughtExceptionHandler(handler);
+        parseThread.setName(String.format("destination = %s , address = %s , EventParser",
+            destination,
+            runningInfo == null ? null : runningInfo.getAddress().toString()));
+        parseThread.start();
+    }
+
+    public void stop() {
+        super.stop();
+
+        stopHeartBeat(); // 先停止心跳
+        parseThread.interrupt(); // 尝试中断
+        eventSink.interrupt();
+        try {
+            parseThread.join();// 等待其结束
+        } catch (InterruptedException e) {
+            // ignore
+        }
+
+        if (binlogParser.isStart()) {
+            binlogParser.stop();
+        }
+        if (transactionBuffer.isStart()) {
+            transactionBuffer.stop();
+        }
+    }
+
+    protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
+                                                                                           InterruptedException {
+        long startTs = -1;
+        boolean enabled = getProfilingEnabled();
+        if (enabled) {
+            startTs = System.currentTimeMillis();
+        }
+
+        boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);
+
+        if (enabled) {
+            this.processingInterval = System.currentTimeMillis() - startTs;
+        }
+
+        if (consumedEventCount.incrementAndGet() < 0) {
+            consumedEventCount.set(0);
+        }
+
+        return result;
+    }
+
+    protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception {
+        long startTs = -1;
+        boolean enabled = getProfilingEnabled();
+        if (enabled) {
+            startTs = System.currentTimeMillis();
+        }
+        CanalEntry.Entry event = binlogParser.parse(bod);
+
+        if (enabled) {
+            this.parsingInterval = System.currentTimeMillis() - startTs;
+        }
+
+        if (parsedEventCount.incrementAndGet() < 0) {
+            parsedEventCount.set(0);
+        }
+        return event;
+    }
+
+    public Boolean getProfilingEnabled() {
+        return profilingEnabled.get();
+    }
+
+    protected LogPosition buildLastTranasctionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
+        for (int i = entries.size() - 1; i > 0; i--) {
+            CanalEntry.Entry entry = entries.get(i);
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
+                LogPosition logPosition = new LogPosition();
+                EntryPosition position = new EntryPosition();
+                position.setJournalName(entry.getHeader().getLogfileName());
+                position.setPosition(entry.getHeader().getLogfileOffset());
+                position.setTimestamp(entry.getHeader().getExecuteTime());
+                logPosition.setPostion(position);
+
+                LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
+                logPosition.setIdentity(identity);
+                return logPosition;
+            }
+        }
+
+        return null;
+    }
+
+    protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
+        LogPosition logPosition = new LogPosition();
+        EntryPosition position = new EntryPosition();
+        position.setJournalName(entry.getHeader().getLogfileName());
+        position.setPosition(entry.getHeader().getLogfileOffset());
+        position.setTimestamp(entry.getHeader().getExecuteTime());
+        logPosition.setPostion(position);
+
+        LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
+        logPosition.setIdentity(identity);
+        return logPosition;
+    }
+
+    protected void processError(Exception e, LogPosition lastPosition, String startBinlogFile, long startPosition) {
+        if (lastPosition != null) {
+            logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
+                lastPosition.getPostion()),
+                e);
+        } else {
+            logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s,%s]",
+                startBinlogFile,
+                startPosition), e);
+        }
+    }
+
+    protected void startHeartBeat(ErosaConnection connection) {
+        lastEntryTime = 0L; // 初始化
+        if (timer == null) {// lazy初始化一下
+            String name = String.format("destination = %s , address = %s , HeartBeatTimeTask",
+                destination,
+                runningInfo == null ? null : runningInfo.getAddress().toString());
+            synchronized (MysqlEventParser.class) {
+                if (timer == null) {
+                    timer = new Timer(name, true);
+                }
+            }
+        }
+
+        if (heartBeatTimerTask == null) {// fixed issue #56，避免重复创建heartbeat线程
+            heartBeatTimerTask = buildHeartBeatTimeTask(connection);
+            Integer interval = detectingIntervalInSeconds;
+            timer.schedule(heartBeatTimerTask, interval * 1000L, interval * 1000L);
+            logger.info("start heart beat.... ");
+        }
+    }
+
+    protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
+        return new TimerTask() {
+
+            public void run() {
+                try {
+                    if (exception == null || lastEntryTime > 0) {
+                        // 如果未出现异常，或者有第一条正常数据
+                        long now = System.currentTimeMillis();
+                        long inteval = (now - lastEntryTime) / 1000;
+                        if (inteval >= detectingIntervalInSeconds) {
+                            Header.Builder headerBuilder = Header.newBuilder();
+                            headerBuilder.setExecuteTime(now);
+                            Entry.Builder entryBuilder = Entry.newBuilder();
+                            entryBuilder.setHeader(headerBuilder.build());
+                            entryBuilder.setEntryType(EntryType.HEARTBEAT);
+                            Entry entry = entryBuilder.build();
+                            // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
+                            consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
+                        }
+                    }
+
+                } catch (Throwable e) {
+                    logger.warn("heartBeat run failed " + ExceptionUtils.getStackTrace(e));
+                }
+            }
+
+        };
+    }
+
+    protected void stopHeartBeat() {
+        lastEntryTime = 0L; // 初始化
+        if (timer != null) {
+            timer.cancel();
+            timer = null;
+        }
+        heartBeatTimerTask = null;
+    }
+
+    public void setEventFilter(CanalEventFilter eventFilter) {
+        this.eventFilter = eventFilter;
+    }
+
+    public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
+        this.eventBlackFilter = eventBlackFilter;
+    }
+
+    public Long getParsedEventCount() {
+        return parsedEventCount.get();
+    }
+
+    public Long getConsumedEventCount() {
+        return consumedEventCount.get();
+    }
+
+    public void setProfilingEnabled(boolean profilingEnabled) {
+        this.profilingEnabled = new AtomicBoolean(profilingEnabled);
+    }
+
+    public long getParsingInterval() {
+        return parsingInterval;
+    }
+
+    public long getProcessingInterval() {
+        return processingInterval;
+    }
+
+    public void setEventSink(CanalEventSink<List<CanalEntry.Entry>> eventSink) {
+        this.eventSink = eventSink;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void setBinlogParser(BinlogParser binlogParser) {
+        this.binlogParser = binlogParser;
+    }
+
+    public BinlogParser getBinlogParser() {
+        return binlogParser;
+    }
+
+    public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
+        this.alarmHandler = alarmHandler;
+    }
+
+    public CanalAlarmHandler getAlarmHandler() {
+        return this.alarmHandler;
+    }
+
+    public void setLogPositionManager(CanalLogPositionManager logPositionManager) {
+        this.logPositionManager = logPositionManager;
+    }
+
+    public void setTransactionSize(int transactionSize) {
+        this.transactionSize = transactionSize;
+    }
+
+    public CanalLogPositionManager getLogPositionManager() {
+        return logPositionManager;
+    }
+
+    public void setDetectingEnable(boolean detectingEnable) {
+        this.detectingEnable = detectingEnable;
+    }
+
+    public void setDetectingIntervalInSeconds(Integer detectingIntervalInSeconds) {
+        this.detectingIntervalInSeconds = detectingIntervalInSeconds;
+    }
+
+    public Throwable getException() {
+        return exception;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java

@@ -0,0 +1,17 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+
+/**
+ * 解析binlog的接口
+ * 
+ * @author: yuanzu Date: 12-9-20 Time: 下午8:46
+ */
+public interface BinlogParser<T> extends CanalLifeCycle {
+
+    CanalEntry.Entry parse(T event) throws CanalParseException;
+
+    void reset();
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java

@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.io.IOException;
+
+/**
+ * 通用的Erosa的链接接口, 用于一般化处理mysql/oracle的解析过程
+ * 
+ * @author: yuanzu Date: 12-9-20 Time: 下午2:47
+ */
+public interface ErosaConnection {
+
+    public void connect() throws IOException;
+
+    public void reconnect() throws IOException;
+
+    public void disconnect() throws IOException;
+
+    public boolean isConnected();
+
+    /**
+     * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据
+     */
+    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
+
+    public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
+
+    public void dump(long timestamp, SinkFunction func) throws IOException;
+
+    ErosaConnection fork();
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java

@@ -0,0 +1,164 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.store.CanalStoreException;
+
+/**
+ * 缓冲event队列，提供按事务刷新数据的机制
+ * 
+ * @author jianghang 2012-12-6 上午11:05:12
+ * @version 1.0.0
+ */
+public class EventTransactionBuffer extends AbstractCanalLifeCycle {
+
+    private static final long        INIT_SQEUENCE = -1;
+    private int                      bufferSize    = 1024;
+    private int                      indexMask;
+    private CanalEntry.Entry[]       entries;
+
+    private AtomicLong               putSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前put操作最后一次写操作发生的位置
+    private AtomicLong               flushSequence = new AtomicLong(INIT_SQEUENCE); // 代表满足flush条件后最后一次数据flush的时间
+
+    private TransactionFlushCallback flushCallback;
+
+    public EventTransactionBuffer(){
+
+    }
+
+    public EventTransactionBuffer(TransactionFlushCallback flushCallback){
+        this.flushCallback = flushCallback;
+    }
+
+    public void start() throws CanalStoreException {
+        super.start();
+        if (Integer.bitCount(bufferSize) != 1) {
+            throw new IllegalArgumentException("bufferSize must be a power of 2");
+        }
+
+        Assert.notNull(flushCallback, "flush callback is null!");
+        indexMask = bufferSize - 1;
+        entries = new CanalEntry.Entry[bufferSize];
+    }
+
+    public void stop() throws CanalStoreException {
+        putSequence.set(INIT_SQEUENCE);
+        flushSequence.set(INIT_SQEUENCE);
+
+        entries = null;
+        super.stop();
+    }
+
+    public void add(List<CanalEntry.Entry> entrys) throws InterruptedException {
+        for (CanalEntry.Entry entry : entrys) {
+            add(entry);
+        }
+    }
+
+    public void add(CanalEntry.Entry entry) throws InterruptedException {
+        switch (entry.getEntryType()) {
+            case TRANSACTIONBEGIN:
+                flush();// 刷新上一次的数据
+                put(entry);
+                break;
+            case TRANSACTIONEND:
+                put(entry);
+                flush();
+                break;
+            case ROWDATA:
+                put(entry);
+                // 针对非DML的数据，直接输出，不进行buffer控制
+                EventType eventType = entry.getHeader().getEventType();
+                if (eventType != null && !isDml(eventType)) {
+                    flush();
+                }
+                break;
+            default:
+                break;
+        }
+    }
+
+    public void reset() {
+        putSequence.set(INIT_SQEUENCE);
+        flushSequence.set(INIT_SQEUENCE);
+    }
+
+    private void put(CanalEntry.Entry data) throws InterruptedException {
+        // 首先检查是否有空位
+        if (checkFreeSlotAt(putSequence.get() + 1)) {
+            long current = putSequence.get();
+            long next = current + 1;
+
+            // 先写数据，再更新对应的cursor,并发度高的情况，putSequence会被get请求可见，拿出了ringbuffer中的老的Entry值
+            entries[getIndex(next)] = data;
+            putSequence.set(next);
+        } else {
+            flush();// buffer区满了，刷新一下
+            put(data);// 继续加一下新数据
+        }
+    }
+
+    private void flush() throws InterruptedException {
+        long start = this.flushSequence.get() + 1;
+        long end = this.putSequence.get();
+
+        if (start <= end) {
+            List<CanalEntry.Entry> transaction = new ArrayList<CanalEntry.Entry>();
+            for (long next = start; next <= end; next++) {
+                transaction.add(this.entries[getIndex(next)]);
+            }
+
+            flushCallback.flush(transaction);
+            flushSequence.set(end);// flush成功后，更新flush位置
+        }
+    }
+
+    /**
+     * 查询是否有空位
+     */
+    private boolean checkFreeSlotAt(final long sequence) {
+        final long wrapPoint = sequence - bufferSize;
+        if (wrapPoint > flushSequence.get()) { // 刚好追上一轮
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    private int getIndex(long sequcnce) {
+        return (int) sequcnce & indexMask;
+    }
+
+    private boolean isDml(EventType eventType) {
+        return eventType == EventType.INSERT || eventType == EventType.UPDATE || eventType == EventType.DELETE;
+    }
+
+    // ================ setter / getter ==================
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+    public void setFlushCallback(TransactionFlushCallback flushCallback) {
+        this.flushCallback = flushCallback;
+    }
+
+    /**
+     * 事务刷新机制
+     * 
+     * @author jianghang 2012-12-6 上午11:57:38
+     * @version 1.0.0
+     */
+    public static interface TransactionFlushCallback {
+
+        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/HeartBeatCallback.java

@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+/**
+ * 提供mysql heartBeat心跳数据的callback机制
+ * 
+ * @author jianghang 2012-6-26 下午04:49:56
+ * @version 1.0.0
+ */
+public interface HeartBeatCallback {
+
+    /**
+     * 心跳发送成功
+     */
+    public void onSuccess(long costTime);
+
+    /**
+     * 心跳发送失败
+     */
+    public void onFailed(Throwable e);
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/SinkFunction.java

@@ -0,0 +1,12 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+/**
+ * receive parsed bytes , 用于处理要解析的数据块
+ * 
+ * @author: yuanzu Date: 12-9-20 Time: 下午2:50
+ */
+
+public interface SinkFunction<EVENT> {
+
+    public boolean sink(EVENT event);
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java

@@ -0,0 +1,122 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+
+/**
+ * 描述数据meta对象,mysql binlog中对应的{@linkplain TableMapLogEvent}包含的信息不全
+ * 
+ * <pre>
+ * 1. 主键信息
+ * 2. column name
+ * 3. unsigned字段
+ * </pre>
+ * 
+ * @author jianghang 2013-1-18 下午12:24:59
+ * @version 1.0.0
+ */
+public class TableMeta {
+
+    private String          fullName; // schema.table
+    private List<FieldMeta> fileds;
+
+    public TableMeta(String fullName, List<FieldMeta> fileds){
+        this.fullName = fullName;
+        this.fileds = fileds;
+    }
+
+    public String getFullName() {
+        return fullName;
+    }
+
+    public void setFullName(String fullName) {
+        this.fullName = fullName;
+    }
+
+    public List<FieldMeta> getFileds() {
+        return fileds;
+    }
+
+    public void setFileds(List<FieldMeta> fileds) {
+        this.fileds = fileds;
+    }
+
+    public static class FieldMeta {
+
+        private String columnName;
+        private String columnType;
+        private String isNullable;
+        private String iskey;
+        private String defaultValue;
+        private String extra;
+
+        public String getColumnName() {
+            return columnName;
+        }
+
+        public void setColumnName(String columnName) {
+            this.columnName = columnName;
+        }
+
+        public String getColumnType() {
+            return columnType;
+        }
+
+        public void setColumnType(String columnType) {
+            this.columnType = columnType;
+        }
+
+        public String getIsNullable() {
+            return isNullable;
+        }
+
+        public void setIsNullable(String isNullable) {
+            this.isNullable = isNullable;
+        }
+
+        public String getIskey() {
+            return iskey;
+        }
+
+        public void setIskey(String iskey) {
+            this.iskey = iskey;
+        }
+
+        public String getDefaultValue() {
+            return defaultValue;
+        }
+
+        public void setDefaultValue(String defaultValue) {
+            this.defaultValue = defaultValue;
+        }
+
+        public String getExtra() {
+            return extra;
+        }
+
+        public void setExtra(String extra) {
+            this.extra = extra;
+        }
+
+        public boolean isUnsigned() {
+            return StringUtils.containsIgnoreCase(columnType, "unsigned");
+        }
+
+        public boolean isKey() {
+            return StringUtils.equalsIgnoreCase(iskey, "PRI");
+        }
+
+        public boolean isNullable() {
+            return StringUtils.equalsIgnoreCase(isNullable, "YES");
+        }
+
+        public String toString() {
+            return "FieldMeta [columnName=" + columnName + ", columnType=" + columnType + ", defaultValue="
+                   + defaultValue + ", extra=" + extra + ", isNullable=" + isNullable + ", iskey=" + iskey + "]";
+        }
+
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java

@@ -0,0 +1,57 @@
+package com.alibaba.otter.canal.parse.inbound.group;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+
+/**
+ * 组合多个EventParser进行合并处理，group只是做为一个delegate处理
+ * 
+ * @author jianghang 2012-10-16 上午11:23:14
+ * @version 1.0.0
+ */
+public class GroupEventParser extends AbstractCanalLifeCycle implements CanalEventParser {
+
+    private List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
+
+    public void start() {
+        super.start();
+        // 统一启动
+        for (CanalEventParser eventParser : eventParsers) {
+            if (!eventParser.isStart()) {
+                eventParser.start();
+            }
+        }
+    }
+
+    public void stop() {
+        super.stop();
+        // 统一关闭
+        for (CanalEventParser eventParser : eventParsers) {
+            if (eventParser.isStart()) {
+                eventParser.stop();
+            }
+        }
+    }
+
+    public void setEventParsers(List<CanalEventParser> eventParsers) {
+        this.eventParsers = eventParsers;
+    }
+
+    public void addEventParser(CanalEventParser eventParser) {
+        if (!eventParsers.contains(eventParser)) {
+            eventParsers.add(eventParser);
+        }
+    }
+
+    public void removeEventParser(CanalEventParser eventParser) {
+        eventParsers.remove(eventParser);
+    }
+
+    public List<CanalEventParser> getEventParsers() {
+        return eventParsers;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java

@@ -0,0 +1,94 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.nio.charset.Charset;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
+import com.alibaba.otter.canal.parse.inbound.BinlogParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
+
+public abstract class AbstractMysqlEventParser extends AbstractEventParser {
+
+    protected final Logger      logger                  = LoggerFactory.getLogger(this.getClass());
+    protected static final long BINLOG_START_OFFEST     = 4L;
+
+    // 编码信息
+    protected byte              connectionCharsetNumber = (byte) 33;
+    protected Charset           connectionCharset       = Charset.forName("UTF-8");
+    protected boolean           filterQueryDcl          = false;
+    protected boolean           filterQueryDml          = false;
+    protected boolean           filterQueryDdl          = false;
+    protected boolean           filterTableError        = false;
+
+    protected BinlogParser buildParser() {
+        LogEventConvert convert = new LogEventConvert();
+        if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
+            convert.setNameFilter((AviaterRegexFilter) eventFilter);
+        }
+
+        if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
+            convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
+        }
+
+        convert.setCharset(connectionCharset);
+        convert.setFilterQueryDcl(filterQueryDcl);
+        convert.setFilterQueryDml(filterQueryDml);
+        convert.setFilterQueryDdl(filterQueryDdl);
+        convert.setFilterTableError(filterTableError);
+        return convert;
+    }
+
+    public void setEventFilter(CanalEventFilter eventFilter) {
+        super.setEventFilter(eventFilter);
+
+        // 触发一下filter变更
+        if (eventFilter != null && eventFilter instanceof AviaterRegexFilter && binlogParser instanceof LogEventConvert) {
+            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
+        }
+    }
+
+    public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
+        super.setEventBlackFilter(eventBlackFilter);
+
+        // 触发一下filter变更
+        if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter
+            && binlogParser instanceof LogEventConvert) {
+            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
+        }
+    }
+
+    // ============================ setter / getter =========================
+
+    public void setConnectionCharsetNumber(byte connectionCharsetNumber) {
+        this.connectionCharsetNumber = connectionCharsetNumber;
+    }
+
+    public void setConnectionCharset(Charset connectionCharset) {
+        this.connectionCharset = connectionCharset;
+    }
+
+    public void setConnectionCharset(String connectionCharset) {
+        this.connectionCharset = Charset.forName(connectionCharset);
+    }
+
+    public void setFilterQueryDcl(boolean filterQueryDcl) {
+        this.filterQueryDcl = filterQueryDcl;
+    }
+
+    public void setFilterQueryDml(boolean filterQueryDml) {
+        this.filterQueryDml = filterQueryDml;
+    }
+
+    public void setFilterQueryDdl(boolean filterQueryDdl) {
+        this.filterQueryDdl = filterQueryDdl;
+    }
+
+    public void setFilterTableError(boolean filterTableError) {
+        this.filterTableError = filterTableError;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/DbsyncMysqlEventParser.java

@@ -0,0 +1,5 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+public class DbsyncMysqlEventParser {
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java

@@ -0,0 +1,244 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.SinkFunction;
+import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue;
+import com.taobao.tddl.dbsync.binlog.FileLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.LogPosition;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+
+/**
+ * local bin log connection (not real connection)
+ * 
+ * @author yuanzu Date: 12-9-27 Time: 下午6:14
+ */
+public class LocalBinLogConnection implements ErosaConnection {
+
+    private static final Logger logger     = LoggerFactory.getLogger(LocalBinLogConnection.class);
+    private BinLogFileQueue     binlogs    = null;
+    private boolean             needWait;
+    private String              directory;
+    private int                 bufferSize = 16 * 1024;
+    private boolean             running    = false;
+
+    public LocalBinLogConnection(){
+    }
+
+    public LocalBinLogConnection(String directory, boolean needWait){
+        this.needWait = needWait;
+        this.directory = directory;
+    }
+
+    @Override
+    public void connect() throws IOException {
+        if (this.binlogs == null) {
+            this.binlogs = new BinLogFileQueue(this.directory);
+        }
+        this.running = true;
+    }
+
+    @Override
+    public void reconnect() throws IOException {
+        disconnect();
+        connect();
+    }
+
+    @Override
+    public void disconnect() throws IOException {
+        this.running = false;
+        if (this.binlogs != null) {
+            this.binlogs.destory();
+        }
+        this.binlogs = null;
+        this.running = false;
+    }
+
+    public boolean isConnected() {
+        return running;
+    }
+
+    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+    }
+
+    public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+        File current = new File(directory, binlogfilename);
+
+        FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
+        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+        LogContext context = new LogContext();
+        try {
+            fetcher.open(current, binlogPosition);
+            context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
+            while (running) {
+                boolean needContinue = true;
+                LogEvent event;
+                while (fetcher.fetch()) {
+                    event = decoder.decode(fetcher, context);
+                    if (event == null) {
+                        throw new CanalParseException("parse failed");
+                    }
+
+                    if (!func.sink(event)) {
+                        needContinue = false;
+                        break;
+                    }
+
+                    // do {
+                    // event = decoder.decode(fetcher, context);
+                    // if (event != null && !func.sink(event)) {
+                    // needContinue = false;
+                    // break;
+                    // }
+                    // } while (event != null);
+                }
+
+                if (needContinue) {// 读取下一个
+                    fetcher.close(); // 关闭上一个文件
+
+                    File nextFile;
+                    if (needWait) {
+                        nextFile = binlogs.waitForNextFile(current);
+                    } else {
+                        nextFile = binlogs.getNextFile(current);
+                    }
+
+                    if (nextFile == null) {
+                        break;
+                    }
+
+                    current = nextFile;
+
+                    fetcher.open(current);
+                    context.setLogPosition(new LogPosition(nextFile.getName()));
+                } else {
+                    break;// 跳出
+                }
+            }
+        } catch (InterruptedException e) {
+            logger.warn("LocalBinLogConnection dump interrupted");
+        } finally {
+            if (fetcher != null) {
+                fetcher.close();
+            }
+        }
+    }
+
+    public void dump(long timestampMills, SinkFunction func) throws IOException {
+        List<File> currentBinlogs = binlogs.currentBinlogs();
+        File current = currentBinlogs.get(currentBinlogs.size() - 1);
+        long timestampSeconds = timestampMills / 1000;
+
+        String binlogFilename = null;
+        long binlogFileOffset = 0;
+
+        FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
+        LogDecoder decoder = new LogDecoder();
+        decoder.handle(LogEvent.QUERY_EVENT);
+        decoder.handle(LogEvent.XID_EVENT);
+        LogContext context = new LogContext();
+        try {
+            fetcher.open(current);
+            context.setLogPosition(new LogPosition(current.getName()));
+            while (running) {
+                boolean needContinue = true;
+                String lastXidLogFilename = current.getName();
+                long lastXidLogFileOffset = 0;
+
+                binlogFilename = lastXidLogFilename;
+                binlogFileOffset = lastXidLogFileOffset;
+                L: while (fetcher.fetch()) {
+                    LogEvent event;
+                    do {
+                        event = decoder.decode(fetcher, context);
+                        if (event != null) {
+                            if (event.getWhen() > timestampSeconds) {
+                                break L;
+                            }
+
+                            needContinue = false;
+                            if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
+                                if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
+                                    binlogFilename = lastXidLogFilename;
+                                    binlogFileOffset = lastXidLogFileOffset;
+                                } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
+                                    lastXidLogFilename = current.getName();
+                                    lastXidLogFileOffset = event.getLogPos();
+                                }
+                            } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
+                                lastXidLogFilename = current.getName();
+                                lastXidLogFileOffset = event.getLogPos();
+                            }
+                        }
+                    } while (event != null);
+                }
+
+                if (needContinue) {// 读取下一个
+                    fetcher.close(); // 关闭上一个文件
+
+                    File nextFile = binlogs.getBefore(current);
+                    if (nextFile == null) {
+                        break;
+                    }
+
+                    current = nextFile;
+                    fetcher.open(current);
+                    context.setLogPosition(new LogPosition(current.getName()));
+                } else {
+                    break;// 跳出
+                }
+            }
+        } finally {
+            if (fetcher != null) {
+                fetcher.close();
+            }
+        }
+
+        dump(binlogFilename, binlogFileOffset, func);
+    }
+
+    public ErosaConnection fork() {
+        LocalBinLogConnection connection = new LocalBinLogConnection();
+
+        connection.setBufferSize(this.bufferSize);
+        connection.setDirectory(this.directory);
+        connection.setNeedWait(this.needWait);
+        return connection;
+    }
+
+    public boolean isNeedWait() {
+        return needWait;
+    }
+
+    public void setNeedWait(boolean needWait) {
+        this.needWait = needWait;
+    }
+
+    public String getDirectory() {
+        return directory;
+    }
+
+    public void setDirectory(String directory) {
+        this.directory = directory;
+    }
+
+    public int getBufferSize() {
+        return bufferSize;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java

@@ -0,0 +1,118 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 基于本地binlog文件的复制
+ * 
+ * @author jianghang 2012-6-21 下午04:07:33
+ * @version 1.0.0
+ */
+public class LocalBinlogEventParser extends AbstractMysqlEventParser implements CanalEventParser {
+
+    // 数据库信息
+    private AuthenticationInfo masterInfo;
+    private EntryPosition      masterPosition;        // binlog信息
+
+    private String             directory;
+    private boolean            needWait   = false;
+    private int                bufferSize = 16 * 1024;
+
+    public LocalBinlogEventParser(){
+        // this.runningInfo = new AuthenticationInfo();
+    }
+
+    @Override
+    protected ErosaConnection buildErosaConnection() {
+        return buildLocalBinLogConnection();
+    }
+
+    public void start() throws CanalParseException {
+        if (runningInfo == null) { // 第一次链接主库
+            runningInfo = masterInfo;
+        }
+
+        super.start();
+    }
+
+    private ErosaConnection buildLocalBinLogConnection() {
+        LocalBinLogConnection connection = new LocalBinLogConnection();
+
+        connection.setBufferSize(this.bufferSize);
+        connection.setDirectory(this.directory);
+        connection.setNeedWait(this.needWait);
+
+        return connection;
+    }
+
+    @Override
+    protected EntryPosition findStartPosition(ErosaConnection connection) {
+        // 处理逻辑
+        // 1. 首先查询上一次解析成功的最后一条记录
+        // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
+        // // a. 无机器切换，直接返回
+        // // b. 存在机器切换，按最后一条记录的stamptime进行查找
+        // 3. 不存在最后一条记录，则从默认的位置开始启动
+        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+        if (logPosition == null) {// 找不到历史成功记录
+            EntryPosition entryPosition = masterPosition;
+
+            // 判断一下是否需要按时间订阅
+            if (StringUtils.isEmpty(entryPosition.getJournalName())) {
+                // 如果没有指定binlogName，尝试按照timestamp进行查找
+                if (entryPosition.getTimestamp() != null) {
+                    return new EntryPosition(entryPosition.getTimestamp());
+                }
+            } else {
+                if (entryPosition.getPosition() != null) {
+                    // 如果指定binlogName + offest，直接返回
+                    return entryPosition;
+                } else {
+                    return new EntryPosition(entryPosition.getTimestamp());
+                }
+            }
+        } else {
+            return logPosition.getPostion();
+        }
+
+        return null;
+    }
+
+    // ========================= setter / getter =========================
+
+    public void setLogPositionManager(CanalLogPositionManager logPositionManager) {
+        this.logPositionManager = logPositionManager;
+    }
+
+    public void setDirectory(String directory) {
+        this.directory = directory;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+    public void setMasterPosition(EntryPosition masterPosition) {
+        this.masterPosition = masterPosition;
+    }
+
+    public void setMasterInfo(AuthenticationInfo masterInfo) {
+        this.masterInfo = masterInfo;
+    }
+
+    public boolean isNeedWait() {
+        return needWait;
+    }
+
+    public void setNeedWait(boolean needWait) {
+        this.needWait = needWait;
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java

@@ -0,0 +1,306 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.SinkFunction;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class MysqlConnection implements ErosaConnection {
+
+    private static final Logger logger       = LoggerFactory.getLogger(MysqlConnection.class);
+
+    private MysqlConnector      connector;
+    private long                slaveId;
+    private Charset             charset      = Charset.forName("UTF-8");
+    private BinlogFormat        binlogFormat = BinlogFormat.ROW;
+
+    public MysqlConnection(){
+    }
+
+    public MysqlConnection(InetSocketAddress address, String username, String password){
+
+        connector = new MysqlConnector(address, username, password);
+    }
+
+    public MysqlConnection(InetSocketAddress address, String username, String password, byte charsetNumber,
+                           String defaultSchema){
+        connector = new MysqlConnector(address, username, password, charsetNumber, defaultSchema);
+    }
+
+    public void connect() throws IOException {
+        connector.connect();
+    }
+
+    public void reconnect() throws IOException {
+        connector.reconnect();
+    }
+
+    public void disconnect() throws IOException {
+        connector.disconnect();
+    }
+
+    public boolean isConnected() {
+        return connector.isConnected();
+    }
+
+    public ResultSetPacket query(String cmd) throws IOException {
+        MysqlQueryExecutor exector = new MysqlQueryExecutor(connector);
+        return exector.query(cmd);
+    }
+
+    public void update(String cmd) throws IOException {
+        MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
+        exector.update(cmd);
+    }
+
+    /**
+     * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
+     */
+    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+        updateSettings();
+
+        sendBinlogDump(binlogfilename, binlogPosition);
+        DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
+        fetcher.start(connector.getChannel());
+        LogDecoder decoder = new LogDecoder();
+        decoder.handle(LogEvent.ROTATE_EVENT);
+        decoder.handle(LogEvent.QUERY_EVENT);
+        decoder.handle(LogEvent.XID_EVENT);
+        LogContext context = new LogContext();
+        while (fetcher.fetch()) {
+            LogEvent event = null;
+            event = decoder.decode(fetcher, context);
+
+            if (event == null) {
+                throw new CanalParseException("parse failed");
+            }
+
+            if (!func.sink(event)) {
+                break;
+            }
+        }
+    }
+
+    public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+        updateSettings();
+        BinlogFormat format = getBinlogFormat();
+        if (!format.isRow()) {
+            logger.warn("binlog_format : {} , it will not have rowData before/after columns", format.toString());
+        }
+
+        sendBinlogDump(binlogfilename, binlogPosition);
+        DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
+        fetcher.start(connector.getChannel());
+        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+        LogContext context = new LogContext();
+        while (fetcher.fetch()) {
+            LogEvent event = null;
+            event = decoder.decode(fetcher, context);
+
+            if (event == null) {
+                throw new CanalParseException("parse failed");
+            }
+
+            if (!func.sink(event)) {
+                break;
+            }
+        }
+    }
+
+    public void dump(long timestamp, SinkFunction func) throws IOException {
+        throw new NullPointerException("Not implement yet");
+    }
+
+    private void sendBinlogDump(String binlogfilename, Long binlogPosition) throws IOException {
+        BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
+        binlogDumpCmd.binlogFileName = binlogfilename;
+        binlogDumpCmd.binlogPosition = binlogPosition;
+        binlogDumpCmd.slaveServerId = this.slaveId;
+        byte[] cmdBody = binlogDumpCmd.toBytes();
+
+        logger.info("COM_BINLOG_DUMP with position:{}", binlogDumpCmd);
+        HeaderPacket binlogDumpHeader = new HeaderPacket();
+        binlogDumpHeader.setPacketBodyLength(cmdBody.length);
+        binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
+        PacketManager.write(connector.getChannel(), new ByteBuffer[] { ByteBuffer.wrap(binlogDumpHeader.toBytes()),
+                ByteBuffer.wrap(cmdBody) });
+    }
+
+    public MysqlConnection fork() {
+        MysqlConnection connection = new MysqlConnection();
+        connection.setCharset(getCharset());
+        connection.setSlaveId(getSlaveId());
+        connection.setConnector(connector.fork());
+        return connection;
+    }
+
+    // ====================== help method ====================
+
+    /**
+     * the settings that will need to be checked or set:<br>
+     * <ol>
+     * <li>wait_timeout</li>
+     * <li>net_write_timeout</li>
+     * <li>net_read_timeout</li>
+     * </ol>
+     * 
+     * @param channel
+     * @throws IOException
+     */
+    private void updateSettings() throws IOException {
+        try {
+            update("set wait_timeout=9999999");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+        try {
+            update("set net_write_timeout=1800");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        try {
+            update("set net_read_timeout=1800");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        try {
+            // 设置服务端返回结果时不做编码转化，直接按照数据库的二进制编码进行发送，由客户端自己根据需求进行编码转化
+            update("set names 'binary'");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        try {
+            // mysql5.6针对checksum支持需要设置session变量
+            // 如果不设置会出现错误： Slave can not handle replication events with the
+            // checksum that master is configured to log
+            // 但也不能乱设置，需要和mysql server的checksum配置一致，不然RotateLogEvent会出现乱码
+            update("set @master_binlog_checksum= '@@global.binlog_checksum'");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        
+        try {
+            // mariadb针对特殊的类型，需要设置session变量
+            update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+    }
+
+    /**
+     * 判断一下是否采用ROW模式
+     */
+    private void loadBinlogFormat() {
+        ResultSetPacket rs = null;
+        try {
+            rs = query("show variables like 'binlog_format'");
+        } catch (IOException e) {
+            throw new CanalParseException(e);
+        }
+
+        List<String> columnValues = rs.getFieldValues();
+        if (columnValues == null || columnValues.size() != 2) {
+            logger.warn("unexpected binlog format query result, this may cause unexpected result, so throw exception to request network to io shutdown.");
+            throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
+        }
+
+        binlogFormat = BinlogFormat.valuesOf(columnValues.get(1));
+        if (binlogFormat == null) {
+            throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
+        }
+    }
+
+    public static enum BinlogFormat {
+
+        STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED");
+
+        public boolean isStatement() {
+            return this == STATEMENT;
+        }
+
+        public boolean isRow() {
+            return this == ROW;
+        }
+
+        public boolean isMixed() {
+            return this == MIXED;
+        }
+
+        private String value;
+
+        private BinlogFormat(String value){
+            this.value = value;
+        }
+
+        public static BinlogFormat valuesOf(String value) {
+            BinlogFormat[] formats = values();
+            for (BinlogFormat format : formats) {
+                if (format.value.equalsIgnoreCase(value)) {
+                    return format;
+                }
+            }
+            return null;
+        }
+    }
+
+    // ================== setter / getter ===================
+
+    public Charset getCharset() {
+        return charset;
+    }
+
+    public void setCharset(Charset charset) {
+        this.charset = charset;
+    }
+
+    public long getSlaveId() {
+        return slaveId;
+    }
+
+    public void setSlaveId(long slaveId) {
+        this.slaveId = slaveId;
+    }
+
+    public MysqlConnector getConnector() {
+        return connector;
+    }
+
+    public void setConnector(MysqlConnector connector) {
+        this.connector = connector;
+    }
+
+    public BinlogFormat getBinlogFormat() {
+        if (binlogFormat == null) {
+            synchronized (this) {
+                loadBinlogFormat();
+            }
+        }
+
+        return binlogFormat;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java

@@ -0,0 +1,738 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.CanalHASwitchable;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.ha.CanalHAController;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback;
+import com.alibaba.otter.canal.parse.inbound.SinkFunction;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * 基于向mysql server复制binlog实现
+ * 
+ * <pre>
+ * 1. 自身不控制mysql主备切换，由ha机制来控制. 比如接入tddl/cobar/自身心跳包成功率
+ * 2. 切换机制
+ * </pre>
+ * 
+ * @author jianghang 2012-6-21 下午04:06:32
+ * @version 1.0.0
+ */
+public class MysqlEventParser extends AbstractMysqlEventParser implements CanalEventParser, CanalHASwitchable {
+
+    private CanalHAController  haController                      = null;
+
+    private int                defaultConnectionTimeoutInSeconds = 30;       // sotimeout
+    private int                receiveBufferSize                 = 64 * 1024;
+    private int                sendBufferSize                    = 64 * 1024;
+    // 数据库信息
+    private AuthenticationInfo masterInfo;                                   // 主库
+    private AuthenticationInfo standbyInfo;                                  // 备库
+    // binlog信息
+    private EntryPosition      masterPosition;
+    private EntryPosition      standbyPosition;
+    private long               slaveId;                                      // 链接到mysql的slave
+    // 心跳检查信息
+    private String             detectingSQL;                                 // 心跳sql
+    private MysqlConnection    metaConnection;                               // 查询meta信息的链接
+    private TableMetaCache     tableMetaCache;                               // 对应meta
+                                                                              // cache
+    private int                fallbackIntervalInSeconds         = 60;       // 切换回退时间
+
+    // 心跳检查
+
+    protected ErosaConnection buildErosaConnection() {
+        return buildMysqlConnection(this.runningInfo);
+    }
+
+    protected void preDump(ErosaConnection connection) {
+        if (!(connection instanceof MysqlConnection)) {
+            throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
+        }
+
+        if (binlogParser != null && binlogParser instanceof LogEventConvert) {
+            metaConnection = (MysqlConnection) connection.fork();
+            try {
+                metaConnection.connect();
+            } catch (IOException e) {
+                throw new CanalParseException(e);
+            }
+
+            tableMetaCache = new TableMetaCache(metaConnection);
+            ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
+        }
+    }
+
+    protected void afterDump(ErosaConnection connection) {
+        super.afterDump(connection);
+
+        if (!(connection instanceof MysqlConnection)) {
+            throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
+        }
+
+        if (metaConnection != null) {
+            try {
+                metaConnection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+    }
+
+    public void start() throws CanalParseException {
+        if (runningInfo == null) { // 第一次链接主库
+            runningInfo = masterInfo;
+        }
+
+        super.start();
+    }
+
+    public void stop() throws CanalParseException {
+        if (metaConnection != null) {
+            try {
+                metaConnection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+
+        if (tableMetaCache != null) {
+            tableMetaCache.clearTableMeta();
+        }
+
+        super.stop();
+    }
+
+    protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
+        if (!(connection instanceof MysqlConnection)) {
+            throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
+        }
+
+        // 开始mysql心跳sql
+        if (detectingEnable && StringUtils.isNotBlank(detectingSQL)) {
+            return new MysqlDetectingTimeTask((MysqlConnection) connection.fork());
+        } else {
+            return super.buildHeartBeatTimeTask(connection);
+        }
+
+    }
+
+    protected void stopHeartBeat() {
+        super.stopHeartBeat();
+
+        if (heartBeatTimerTask != null) {
+
+            MysqlConnection mysqlConnection = ((MysqlDetectingTimeTask) heartBeatTimerTask).getMysqlConnection();
+            try {
+                mysqlConnection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect heartbeat connection for address:{}", mysqlConnection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+    }
+
+    /**
+     * 心跳信息
+     * 
+     * @author jianghang 2012-7-6 下午02:50:15
+     * @version 1.0.0
+     */
+    class MysqlDetectingTimeTask extends TimerTask {
+
+        private boolean         reconnect = false;
+        private MysqlConnection mysqlConnection;
+
+        public MysqlDetectingTimeTask(MysqlConnection mysqlConnection){
+            this.mysqlConnection = mysqlConnection;
+        }
+
+        public void run() {
+            try {
+                if (reconnect) {
+                    reconnect = false;
+                    mysqlConnection.reconnect();
+                } else if (!mysqlConnection.isConnected()) {
+                    mysqlConnection.connect();
+                }
+                Long startTime = System.currentTimeMillis();
+
+                // 可能心跳sql为select 1
+                if (StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "select")
+                    || StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "show")
+                    || StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "explain")
+                    || StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "desc")) {
+                    mysqlConnection.query(detectingSQL);
+                } else {
+                    mysqlConnection.update(detectingSQL);
+                }
+
+                Long costTime = System.currentTimeMillis() - startTime;
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onSuccess(costTime);
+                }
+            } catch (SocketTimeoutException e) {
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onFailed(e);
+                }
+                reconnect = true;
+                logger.warn("connect failed by " + ExceptionUtils.getStackTrace(e));
+            } catch (IOException e) {
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onFailed(e);
+                }
+                reconnect = true;
+                logger.warn("connect failed by " + ExceptionUtils.getStackTrace(e));
+            } catch (Throwable e) {
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onFailed(e);
+                }
+                reconnect = true;
+                logger.warn("connect failed by " + ExceptionUtils.getStackTrace(e));
+            }
+
+        }
+
+        public MysqlConnection getMysqlConnection() {
+            return mysqlConnection;
+        }
+    }
+
+    // 处理主备切换的逻辑
+    public void doSwitch() {
+        AuthenticationInfo newRunningInfo = (runningInfo.equals(masterInfo) ? standbyInfo : masterInfo);
+        this.doSwitch(newRunningInfo);
+    }
+
+    public void doSwitch(AuthenticationInfo newRunningInfo) {
+        // 1. 需要停止当前正在复制的过程
+        // 2. 找到新的position点
+        // 3. 重新建立链接，开始复制数据
+        // 切换ip
+        String alarmMessage = null;
+
+        if (this.runningInfo.equals(newRunningInfo)) {
+            alarmMessage = "same runingInfo switch again : " + runningInfo.getAddress().toString();
+            logger.warn(alarmMessage);
+            return;
+        }
+
+        if (newRunningInfo == null) {
+            alarmMessage = "no standby config, just do nothing, will continue try:"
+                           + runningInfo.getAddress().toString();
+            logger.warn(alarmMessage);
+            sendAlarm(destination, alarmMessage);
+            return;
+        } else {
+            stop();
+            alarmMessage = "try to ha switch, old:" + runningInfo.getAddress().toString() + ", new:"
+                           + newRunningInfo.getAddress().toString();
+            logger.warn(alarmMessage);
+            sendAlarm(destination, alarmMessage);
+            runningInfo = newRunningInfo;
+            start();
+        }
+    }
+
+    // =================== helper method =================
+
+    private MysqlConnection buildMysqlConnection(AuthenticationInfo runningInfo) {
+        MysqlConnection connection = new MysqlConnection(runningInfo.getAddress(),
+            runningInfo.getUsername(),
+            runningInfo.getPassword(),
+            connectionCharsetNumber,
+            runningInfo.getDefaultDatabaseName());
+        connection.getConnector().setReceiveBufferSize(receiveBufferSize);
+        connection.getConnector().setSendBufferSize(sendBufferSize);
+        connection.getConnector().setSoTimeout(defaultConnectionTimeoutInSeconds * 1000);
+        connection.setCharset(connectionCharset);
+        connection.setSlaveId(this.slaveId);
+        return connection;
+    }
+
+    protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
+        EntryPosition startPosition = findStartPositionInternal(connection);
+        if (needTransactionPosition.get()) {
+            logger.warn("prepare to find last position : {}", startPosition.toString());
+            Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
+            if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
+                logger.warn("find new start Transaction Position , old : {} , new : {}",
+                    startPosition.getPosition(),
+                    preTransactionStartPosition);
+                startPosition.setPosition(preTransactionStartPosition);
+            }
+            needTransactionPosition.compareAndSet(true, false);
+        }
+        return startPosition;
+    }
+
+    protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
+        MysqlConnection mysqlConnection = (MysqlConnection) connection;
+        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+        if (logPosition == null) {// 找不到历史成功记录
+            EntryPosition entryPosition = null;
+            if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
+                entryPosition = masterPosition;
+            } else if (standbyInfo != null
+                       && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
+                entryPosition = standbyPosition;
+            }
+
+            if (entryPosition == null) {
+                entryPosition = findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
+            }
+
+            // 判断一下是否需要按时间订阅
+            if (StringUtils.isEmpty(entryPosition.getJournalName())) {
+                // 如果没有指定binlogName，尝试按照timestamp进行查找
+                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
+                    logger.warn("prepare to find start position {}:{}:{}",
+                        new Object[] { "", "", entryPosition.getTimestamp() });
+                    return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
+                } else {
+                    logger.warn("prepare to find start position just show master status");
+                    return findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
+                }
+            } else {
+                if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
+                    // 如果指定binlogName + offest，直接返回
+                    logger.warn("prepare to find start position {}:{}:{}",
+                        new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(), "" });
+                    return entryPosition;
+                } else {
+                    EntryPosition specificLogFilePosition = null;
+                    if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
+                        // 如果指定binlogName +
+                        // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
+                        EntryPosition endPosition = findEndPosition(mysqlConnection);
+                        if (endPosition != null) {
+                            logger.warn("prepare to find start position {}:{}:{}",
+                                new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
+                            specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
+                                entryPosition.getTimestamp(),
+                                endPosition,
+                                entryPosition.getJournalName());
+                        }
+                    }
+
+                    if (specificLogFilePosition == null) {
+                        // position不存在，从文件头开始
+                        entryPosition.setPosition(BINLOG_START_OFFEST);
+                        return entryPosition;
+                    } else {
+                        return specificLogFilePosition;
+                    }
+                }
+            }
+        } else {
+            if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
+                logger.warn("prepare to find start position just last position");
+                return logPosition.getPostion();
+            } else {
+                // 针对切换的情况，考虑回退时间
+                long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
+                logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "",
+                        logPosition.getPostion().getTimestamp() });
+                return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
+            }
+        }
+    }
+
+    // 根据想要的position，可能这个position对应的记录为rowdata，需要找到事务头，避免丢数据
+    // 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
+    private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition)
+                                                                                                                 throws IOException {
+        // 尝试找到一个合适的位置
+        final AtomicBoolean reDump = new AtomicBoolean(false);
+        mysqlConnection.reconnect();
+        mysqlConnection.seek(entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction<LogEvent>() {
+
+            private LogPosition lastPosition;
+
+            public boolean sink(LogEvent event) {
+                try {
+                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+                    if (entry == null) {
+                        return true;
+                    }
+
+                    // 直接查询第一条业务数据，确认是否为事务Begin/End
+                    if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType()
+                        || CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) {
+                        lastPosition = buildLastPosition(entry);
+                        return false;
+                    } else {
+                        reDump.set(true);
+                        lastPosition = buildLastPosition(entry);
+                        return false;
+                    }
+                } catch (Exception e) {
+                    // 上一次记录的poistion可能为一条update/insert/delete变更事件，直接进行dump的话，会缺少tableMap事件，导致tableId未进行解析
+                    processError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
+                    reDump.set(true);
+                    return false;
+                }
+            }
+        });
+        // 针对开始的第一条为非Begin记录，需要从该binlog扫描
+        if (reDump.get()) {
+            final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
+            mysqlConnection.reconnect();
+            mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() {
+
+                private LogPosition lastPosition;
+
+                public boolean sink(LogEvent event) {
+                    try {
+                        CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+                        if (entry == null) {
+                            return true;
+                        }
+
+                        // 直接查询第一条业务数据，确认是否为事务Begin
+                        // 记录一下transaction begin position
+                        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                            && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
+                            preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
+                        }
+
+                        if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
+                            return false;// 退出
+                        }
+
+                        lastPosition = buildLastPosition(entry);
+                    } catch (Exception e) {
+                        processError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
+                        return false;
+                    }
+
+                    return running;
+                }
+            });
+
+            // 判断一下找到的最接近position的事务头的位置
+            if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
+                logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
+                throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
+            }
+            return preTransactionStartPosition.get();
+        } else {
+            return entryPosition.getPosition();
+        }
+    }
+
+    // 根据时间查找binlog位置
+    private EntryPosition findByStartTimeStamp(MysqlConnection mysqlConnection, Long startTimestamp) {
+        EntryPosition endPosition = findEndPosition(mysqlConnection);
+        EntryPosition startPosition = findStartPosition(mysqlConnection);
+        String maxBinlogFileName = endPosition.getJournalName();
+        String minBinlogFileName = startPosition.getJournalName();
+        logger.info("show master status to set search end condition:{} ", endPosition);
+        String startSearchBinlogFile = endPosition.getJournalName();
+        boolean shouldBreak = false;
+        while (running && !shouldBreak) {
+            try {
+                EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
+                    startTimestamp,
+                    endPosition,
+                    startSearchBinlogFile);
+                if (entryPosition == null) {
+                    if (StringUtils.equalsIgnoreCase(minBinlogFileName, startSearchBinlogFile)) {
+                        // 已经找到最早的一个binlog，没必要往前找了
+                        shouldBreak = true;
+                        logger.warn("Didn't find the corresponding binlog files from {} to {}",
+                            minBinlogFileName,
+                            maxBinlogFileName);
+                    } else {
+                        // 继续往前找
+                        int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1));
+                        if (binlogSeqNum <= 1) {
+                            logger.warn("Didn't find the corresponding binlog files");
+                            shouldBreak = true;
+                        } else {
+                            int nextBinlogSeqNum = binlogSeqNum - 1;
+                            String binlogFileNamePrefix = startSearchBinlogFile.substring(0,
+                                startSearchBinlogFile.indexOf(".") + 1);
+                            String binlogFileNameSuffix = String.format("%06d", nextBinlogSeqNum);
+                            startSearchBinlogFile = binlogFileNamePrefix + binlogFileNameSuffix;
+                        }
+                    }
+                } else {
+                    logger.info("found and return:{} in findByStartTimeStamp operation.", entryPosition);
+                    return entryPosition;
+                }
+            } catch (Exception e) {
+                logger.warn("the binlogfile:{} doesn't exist, to continue to search the next binlogfile , caused by {}",
+                    startSearchBinlogFile,
+                    ExceptionUtils.getFullStackTrace(e));
+                int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1));
+                if (binlogSeqNum <= 1) {
+                    logger.warn("Didn't find the corresponding binlog files");
+                    shouldBreak = true;
+                } else {
+                    int nextBinlogSeqNum = binlogSeqNum - 1;
+                    String binlogFileNamePrefix = startSearchBinlogFile.substring(0,
+                        startSearchBinlogFile.indexOf(".") + 1);
+                    String binlogFileNameSuffix = String.format("%06d", nextBinlogSeqNum);
+                    startSearchBinlogFile = binlogFileNamePrefix + binlogFileNameSuffix;
+                }
+            }
+        }
+        // 找不到
+        return null;
+    }
+
+    /**
+     * 查询当前的binlog位置
+     */
+    private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
+        try {
+            ResultSetPacket packet = mysqlConnection.query("show master status");
+            List<String> fields = packet.getFieldValues();
+            if (CollectionUtils.isEmpty(fields)) {
+                throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
+            }
+            EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
+            return endPosition;
+        } catch (IOException e) {
+            throw new CanalParseException("command : 'show master status' has an error!", e);
+        }
+    }
+
+    /**
+     * 查询当前的binlog位置
+     */
+    private EntryPosition findStartPosition(MysqlConnection mysqlConnection) {
+        try {
+            ResultSetPacket packet = mysqlConnection.query("show binlog events limit 1");
+            List<String> fields = packet.getFieldValues();
+            if (CollectionUtils.isEmpty(fields)) {
+                throw new CanalParseException("command : 'show binlog events limit 1' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
+            }
+            EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
+            return endPosition;
+        } catch (IOException e) {
+            throw new CanalParseException("command : 'show binlog events limit 1' has an error!", e);
+        }
+
+    }
+
+    /**
+     * 查询当前的slave视图的binlog位置
+     */
+    @SuppressWarnings("unused")
+    private SlaveEntryPosition findSlavePosition(MysqlConnection mysqlConnection) {
+        try {
+            ResultSetPacket packet = mysqlConnection.query("show slave status");
+            List<FieldPacket> names = packet.getFieldDescriptors();
+            List<String> fields = packet.getFieldValues();
+            if (CollectionUtils.isEmpty(fields)) {
+                return null;
+            }
+
+            int i = 0;
+            Map<String, String> maps = new HashMap<String, String>(names.size(), 1f);
+            for (FieldPacket name : names) {
+                maps.put(name.getName(), fields.get(i));
+                i++;
+            }
+
+            String errno = maps.get("Last_Errno");
+            String slaveIORunning = maps.get("Slave_IO_Running"); // Slave_SQL_Running
+            String slaveSQLRunning = maps.get("Slave_SQL_Running"); // Slave_SQL_Running
+            if ((!"0".equals(errno)) || (!"Yes".equalsIgnoreCase(slaveIORunning))
+                || (!"Yes".equalsIgnoreCase(slaveSQLRunning))) {
+                logger.warn("Ignoring failed slave: " + mysqlConnection.getConnector().getAddress() + ", Last_Errno = "
+                            + errno + ", Slave_IO_Running = " + slaveIORunning + ", Slave_SQL_Running = "
+                            + slaveSQLRunning);
+                return null;
+            }
+
+            String masterHost = maps.get("Master_Host");
+            String masterPort = maps.get("Master_Port");
+            String binlog = maps.get("Master_Log_File");
+            String position = maps.get("Exec_Master_Log_Pos");
+            return new SlaveEntryPosition(binlog, Long.valueOf(position), masterHost, masterPort);
+        } catch (IOException e) {
+            logger.error("find slave position error", e);
+        }
+
+        return null;
+    }
+
+    /**
+     * 根据给定的时间戳，在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。
+     * 针对最后一个binlog会给定endPosition，避免无尽的查询
+     */
+    private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection,
+                                                              final Long startTimestamp,
+                                                              final EntryPosition endPosition,
+                                                              final String searchBinlogFile) {
+
+        final LogPosition logPosition = new LogPosition();
+        try {
+            mysqlConnection.reconnect();
+            // 开始遍历文件
+            mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() {
+
+                private LogPosition lastPosition;
+
+                public boolean sink(LogEvent event) {
+                    EntryPosition entryPosition = null;
+                    try {
+                        CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+                        if (entry == null) {
+                            return true;
+                        }
+
+                        String logfilename = entry.getHeader().getLogfileName();
+                        Long logfileoffset = entry.getHeader().getLogfileOffset();
+                        Long logposTimestamp = entry.getHeader().getExecuteTime();
+
+                        if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
+                            logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] {
+                                    logfilename, logfileoffset, logposTimestamp, startTimestamp });
+                            // 寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
+                            if (logposTimestamp >= startTimestamp) {
+                                return false;
+                            }
+                        }
+
+                        if (StringUtils.equals(endPosition.getJournalName(), logfilename)
+                            && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
+                            return false;
+                        }
+
+                        // 记录一下上一个事务结束的位置，即下一个事务的position
+                        // position = current +
+                        // data.length，代表该事务的下一条offest，避免多余的事务重复
+                        if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
+                            entryPosition = new EntryPosition(logfilename,
+                                logfileoffset + event.getEventLen(),
+                                logposTimestamp);
+                            logger.debug("set {} to be pending start position before finding another proper one...",
+                                entryPosition);
+                            logPosition.setPostion(entryPosition);
+                        }
+
+                        lastPosition = buildLastPosition(entry);
+                    } catch (Exception e) {
+                        processError(e, lastPosition, searchBinlogFile, 4L);
+                    }
+
+                    return running;
+                }
+            });
+
+        } catch (IOException e) {
+            logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
+        }
+
+        if (logPosition.getPostion() != null) {
+            return logPosition.getPostion();
+        } else {
+            return null;
+        }
+    }
+
+    protected Entry parseAndProfilingIfNecessary(LogEvent bod) throws Exception {
+        long startTs = -1;
+        boolean enabled = getProfilingEnabled();
+        if (enabled) {
+            startTs = System.currentTimeMillis();
+        }
+        CanalEntry.Entry event = binlogParser.parse(bod);
+        if (enabled) {
+            this.parsingInterval = System.currentTimeMillis() - startTs;
+        }
+
+        if (parsedEventCount.incrementAndGet() < 0) {
+            parsedEventCount.set(0);
+        }
+        return event;
+    }
+
+    // ===================== setter / getter ========================
+
+    public void setDefaultConnectionTimeoutInSeconds(int defaultConnectionTimeoutInSeconds) {
+        this.defaultConnectionTimeoutInSeconds = defaultConnectionTimeoutInSeconds;
+    }
+
+    public void setReceiveBufferSize(int receiveBufferSize) {
+        this.receiveBufferSize = receiveBufferSize;
+    }
+
+    public void setSendBufferSize(int sendBufferSize) {
+        this.sendBufferSize = sendBufferSize;
+    }
+
+    public void setMasterInfo(AuthenticationInfo masterInfo) {
+        this.masterInfo = masterInfo;
+    }
+
+    public void setStandbyInfo(AuthenticationInfo standbyInfo) {
+        this.standbyInfo = standbyInfo;
+    }
+
+    public void setMasterPosition(EntryPosition masterPosition) {
+        this.masterPosition = masterPosition;
+    }
+
+    public void setStandbyPosition(EntryPosition standbyPosition) {
+        this.standbyPosition = standbyPosition;
+    }
+
+    public void setSlaveId(long slaveId) {
+        this.slaveId = slaveId;
+    }
+
+    public void setDetectingSQL(String detectingSQL) {
+        this.detectingSQL = detectingSQL;
+    }
+
+    public void setDetectingIntervalInSeconds(Integer detectingIntervalInSeconds) {
+        this.detectingIntervalInSeconds = detectingIntervalInSeconds;
+    }
+
+    public void setDetectingEnable(boolean detectingEnable) {
+        this.detectingEnable = detectingEnable;
+    }
+
+    public void setFallbackIntervalInSeconds(int fallbackIntervalInSeconds) {
+        this.fallbackIntervalInSeconds = fallbackIntervalInSeconds;
+    }
+
+    public CanalHAController getHaController() {
+        return haController;
+    }
+
+    public void setHaController(CanalHAController haController) {
+        this.haController = haController;
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/SlaveEntryPosition.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+
+/**
+ * slave status状态的信息
+ * 
+ * @author jianghang 2013-1-23 下午09:42:18
+ * @version 1.0.0
+ */
+public class SlaveEntryPosition extends EntryPosition {
+
+    private static final long serialVersionUID = 5271424551446372093L;
+    private final String      masterHost;
+    private final String      masterPort;
+
+    public SlaveEntryPosition(String fileName, long position, String masterHost, String masterPort){
+        super(fileName, position);
+
+        this.masterHost = masterHost;
+        this.masterPort = masterPort;
+    }
+
+    public String getMasterHost() {
+        return masterHost;
+    }
+
+    public String getMasterPort() {
+        return masterPort;
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java

@@ -0,0 +1,180 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.taobao.tddl.dbsync.binlog.LogFetcher;
+
+/**
+ * 基于socket的logEvent实现
+ * 
+ * @author jianghang 2013-1-14 下午07:39:30
+ * @version 1.0.0
+ */
+public class DirectLogFetcher extends LogFetcher {
+
+    protected static final Logger logger            = LoggerFactory.getLogger(DirectLogFetcher.class);
+
+    /** Command to dump binlog */
+    public static final byte      COM_BINLOG_DUMP   = 18;
+
+    /** Packet header sizes */
+    public static final int       NET_HEADER_SIZE   = 4;
+    public static final int       SQLSTATE_LENGTH   = 5;
+
+    /** Packet offsets */
+    public static final int       PACKET_LEN_OFFSET = 0;
+    public static final int       PACKET_SEQ_OFFSET = 3;
+
+    /** Maximum packet length */
+    public static final int       MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
+
+    private SocketChannel         channel;
+
+    // private BufferedInputStream input;
+
+    public DirectLogFetcher(){
+        super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public DirectLogFetcher(final int initialCapacity){
+        super(initialCapacity, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public DirectLogFetcher(final int initialCapacity, final float growthFactor){
+        super(initialCapacity, growthFactor);
+    }
+
+    public void start(SocketChannel channel) throws IOException {
+        this.channel = channel;
+        // 和mysql driver一样，提供buffer机制，提升读取binlog速度
+        // this.input = new
+        // BufferedInputStream(channel.socket().getInputStream(), 16384);
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
+     */
+    public boolean fetch() throws IOException {
+        try {
+            // Fetching packet header from input.
+            if (!fetch0(0, NET_HEADER_SIZE)) {
+                logger.warn("Reached end of input stream while fetching header");
+                return false;
+            }
+
+            // Fetching the first packet(may a multi-packet).
+            int netlen = getUint24(PACKET_LEN_OFFSET);
+            int netnum = getUint8(PACKET_SEQ_OFFSET);
+            if (!fetch0(NET_HEADER_SIZE, netlen)) {
+                logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
+                return false;
+            }
+
+            // Detecting error code.
+            final int mark = getUint8(NET_HEADER_SIZE);
+            if (mark != 0) {
+                if (mark == 255) // error from master
+                {
+                    // Indicates an error, for example trying to fetch from
+                    // wrong
+                    // binlog position.
+                    position = NET_HEADER_SIZE + 1;
+                    final int errno = getInt16();
+                    String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
+                    String errmsg = getFixString(limit - position);
+                    throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
+                                          + " errmsg = " + errmsg);
+                } else if (mark == 254) {
+                    // Indicates end of stream. It's not clear when this would
+                    // be sent.
+                    logger.warn("Received EOF packet from server, apparent"
+                                + " master disconnected. It's may be duplicate slaveId , check instance config");
+                    return false;
+                } else {
+                    // Should not happen.
+                    throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
+                                          + ", len = " + netlen);
+                }
+            }
+
+            // The first packet is a multi-packet, concatenate the packets.
+            while (netlen == MAX_PACKET_LENGTH) {
+                if (!fetch0(0, NET_HEADER_SIZE)) {
+                    logger.warn("Reached end of input stream while fetching header");
+                    return false;
+                }
+
+                netlen = getUint24(PACKET_LEN_OFFSET);
+                netnum = getUint8(PACKET_SEQ_OFFSET);
+                if (!fetch0(limit, netlen)) {
+                    logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
+                    return false;
+                }
+            }
+
+            // Preparing buffer variables to decoding.
+            origin = NET_HEADER_SIZE + 1;
+            position = origin;
+            limit -= origin;
+            return true;
+        } catch (SocketTimeoutException e) {
+            close(); /* Do cleanup */
+            logger.error("Socket timeout expired, closing connection", e);
+            throw e;
+        } catch (InterruptedIOException e) {
+            close(); /* Do cleanup */
+            logger.info("I/O interrupted while reading from client socket", e);
+            throw e;
+        } catch (ClosedByInterruptException e) {
+            close(); /* Do cleanup */
+            logger.info("I/O interrupted while reading from client socket", e);
+            throw e;
+        } catch (IOException e) {
+            close(); /* Do cleanup */
+            logger.error("I/O error while reading from client socket", e);
+            throw e;
+        }
+    }
+
+    private final boolean fetch0(final int off, final int len) throws IOException {
+        ensureCapacity(off + len);
+
+        ByteBuffer buffer = ByteBuffer.wrap(this.buffer, off, len);
+        while (buffer.hasRemaining()) {
+            int readNum = channel.read(buffer);
+            if (readNum == -1) {
+                throw new IOException("Unexpected End Stream");
+            }
+        }
+
+        // for (int count, n = 0; n < len; n += count) {
+        // if (0 > (count = input.read(buffer, off + n, len - n))) {
+        // // Reached end of input stream
+        // return false;
+        // }
+        // }
+
+        if (limit < off + len) limit = off + len;
+        return true;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
+     */
+    public void close() throws IOException {
+        // do nothing
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java

@@ -0,0 +1,707 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.sql.Types;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
+import com.alibaba.otter.canal.parse.inbound.BinlogParser;
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.protocol.CanalEntry.Pair;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin;
+import com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd;
+import com.alibaba.otter.canal.protocol.CanalEntry.Type;
+import com.google.protobuf.ByteString;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+
+/**
+ * 基于{@linkplain LogEvent}转化为Entry对象的处理
+ * 
+ * @author jianghang 2013-1-17 下午02:41:14
+ * @version 1.0.0
+ */
+public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogParser<LogEvent> {
+
+    public static final String          ISO_8859_1          = "ISO-8859-1";
+    public static final String          UTF_8               = "UTF-8";
+    public static final int             TINYINT_MAX_VALUE   = 256;
+    public static final int             SMALLINT_MAX_VALUE  = 65536;
+    public static final int             MEDIUMINT_MAX_VALUE = 16777216;
+    public static final long            INTEGER_MAX_VALUE   = 4294967296L;
+    public static final BigInteger      BIGINT_MAX_VALUE    = new BigInteger("18446744073709551616");
+    public static final int             version             = 1;
+    public static final String          BEGIN               = "BEGIN";
+    public static final String          COMMIT              = "COMMIT";
+    public static final Logger          logger              = LoggerFactory.getLogger(LogEventConvert.class);
+
+    private volatile AviaterRegexFilter nameFilter;                                                          // 运行时引用可能会有变化，比如规则发生变化时
+    private volatile AviaterRegexFilter nameBlackFilter;
+
+    private TableMetaCache              tableMetaCache;
+    private String                      binlogFileName      = "mysql-bin.000001";
+    private Charset                     charset             = Charset.defaultCharset();
+    private boolean                     filterQueryDcl      = false;
+    private boolean                     filterQueryDml      = false;
+    private boolean                     filterQueryDdl      = false;
+    // 是否跳过table相关的解析异常,比如表不存在或者列数量不匹配,issue 92
+    private boolean                     filterTableError    = false;
+
+    public Entry parse(LogEvent logEvent) throws CanalParseException {
+        if (logEvent == null || logEvent instanceof UnknownLogEvent) {
+            return null;
+        }
+
+        int eventType = logEvent.getHeader().getType();
+        switch (eventType) {
+            case LogEvent.ROTATE_EVENT:
+                binlogFileName = ((RotateLogEvent) logEvent).getFilename();
+                break;
+            case LogEvent.QUERY_EVENT:
+                return parseQueryEvent((QueryLogEvent) logEvent);
+            case LogEvent.XID_EVENT:
+                return parseXidEvent((XidLogEvent) logEvent);
+            case LogEvent.TABLE_MAP_EVENT:
+                break;
+            case LogEvent.WRITE_ROWS_EVENT_V1:
+            case LogEvent.WRITE_ROWS_EVENT:
+                return parseRowsEvent((WriteRowsLogEvent) logEvent);
+            case LogEvent.UPDATE_ROWS_EVENT_V1:
+            case LogEvent.UPDATE_ROWS_EVENT:
+                return parseRowsEvent((UpdateRowsLogEvent) logEvent);
+            case LogEvent.DELETE_ROWS_EVENT_V1:
+            case LogEvent.DELETE_ROWS_EVENT:
+                return parseRowsEvent((DeleteRowsLogEvent) logEvent);
+            case LogEvent.ROWS_QUERY_LOG_EVENT:
+                return parseRowsQueryEvent((RowsQueryLogEvent) logEvent);
+            case LogEvent.ANNOTATE_ROWS_EVENT:
+                return parseAnnotateRowsEvent((AnnotateRowsEvent) logEvent);
+            case LogEvent.USER_VAR_EVENT:
+                return parseUserVarLogEvent((UserVarLogEvent) logEvent);
+            case LogEvent.INTVAR_EVENT:
+                return parseIntrvarLogEvent((IntvarLogEvent) logEvent);
+            case LogEvent.RAND_EVENT:
+                return parseRandLogEvent((RandLogEvent) logEvent);
+            default:
+                break;
+        }
+
+        return null;
+    }
+
+    public void reset() {
+        // do nothing
+        binlogFileName = "mysql-bin.000001";
+        if (tableMetaCache != null) {
+            tableMetaCache.clearTableMeta();
+        }
+    }
+
+    private Entry parseQueryEvent(QueryLogEvent event) {
+        String queryString = event.getQuery();
+        if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
+            TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
+            Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
+            return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
+        } else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
+            TransactionEnd transactionEnd = createTransactionEnd(0L); // MyISAM可能不会有xid事件
+            Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
+            return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
+        } else {
+            // DDL语句处理
+            DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
+
+            String schemaName = event.getDbName();
+            if (StringUtils.isNotEmpty(result.getSchemaName())) {
+                schemaName = result.getSchemaName();
+            }
+
+            String tableName = result.getTableName();
+            EventType type = EventType.QUERY;
+            // fixed issue https://github.com/alibaba/canal/issues/58
+            if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE
+                || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE
+                || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX
+                || result.getType() == EventType.DINDEX) { // 针对DDL类型
+
+                if (filterQueryDdl) {
+                    return null;
+                }
+
+                type = result.getType();
+                if (StringUtils.isEmpty(tableName)
+                    || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
+                    // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
+                    throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: "
+                                                  + queryString + " , and DdlResult: " + result.toString());
+                    // return null;
+                } else {
+                    // check name filter
+                    String name = schemaName + "." + tableName;
+                    if (nameFilter != null && !nameFilter.filter(name)) {
+                        if (result.getType() == EventType.RENAME) {
+                            // rename校验只要源和目标满足一个就进行操作
+                            if (nameFilter != null
+                                && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
+                                return null;
+                            }
+                        } else {
+                            // 其他情况返回null
+                            return null;
+                        }
+                    }
+
+                    if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
+                        if (result.getType() == EventType.RENAME) {
+                            // rename校验只要源和目标满足一个就进行操作
+                            if (nameBlackFilter != null
+                                && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
+                                return null;
+                            }
+                        } else {
+                            // 其他情况返回null
+                            return null;
+                        }
+                    }
+                }
+            } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE
+                       || result.getType() == EventType.DELETE) {
+                // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
+                if (filterQueryDml) {
+                    return null;
+                }
+            } else if (filterQueryDcl) {
+                return null;
+            }
+
+            // 更新下table meta cache
+            if (tableMetaCache != null
+                && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
+                if (StringUtils.isNotEmpty(tableName)) {
+                    // 如果解析到了正确的表信息，则根据全名进行清除
+                    tableMetaCache.clearTableMeta(schemaName, tableName);
+                } else {
+                    // 如果无法解析正确的表信息，则根据schema进行清除
+                    tableMetaCache.clearTableMetaWithSchemaName(schemaName);
+                }
+            }
+
+            Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
+            RowChange.Builder rowChangeBuider = RowChange.newBuilder();
+            if (result.getType() != EventType.QUERY) {
+                rowChangeBuider.setIsDdl(true);
+            }
+            rowChangeBuider.setSql(queryString);
+            if (StringUtils.isNotEmpty(event.getDbName())) {// 可能为空
+                rowChangeBuider.setDdlSchemaName(event.getDbName());
+            }
+            rowChangeBuider.setEventType(result.getType());
+            return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
+        }
+    }
+
+    private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+        // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
+        String queryString = null;
+        try {
+            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
+            return buildQueryEntry(queryString, event.getHeader());
+        } catch (UnsupportedEncodingException e) {
+            throw new CanalParseException(e);
+        }
+    }
+
+    private Entry parseAnnotateRowsEvent(AnnotateRowsEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+        // mariaDb支持，需要设置binlog_annotate_row_events=true，可详细打印原始DML语句
+        String queryString = null;
+        try {
+            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
+            return buildQueryEntry(queryString, event.getHeader());
+        } catch (UnsupportedEncodingException e) {
+            throw new CanalParseException(e);
+        }
+    }
+
+    private Entry parseUserVarLogEvent(UserVarLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+
+        return buildQueryEntry(event.getQuery(), event.getHeader());
+    }
+
+    private Entry parseIntrvarLogEvent(IntvarLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+
+        return buildQueryEntry(event.getQuery(), event.getHeader());
+    }
+
+    private Entry parseRandLogEvent(RandLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+
+        return buildQueryEntry(event.getQuery(), event.getHeader());
+    }
+
+    private Entry parseXidEvent(XidLogEvent event) {
+        TransactionEnd transactionEnd = createTransactionEnd(event.getXid());
+        Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
+        return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
+    }
+
+    private Entry parseRowsEvent(RowsLogEvent event) {
+        try {
+            TableMapLogEvent table = event.getTable();
+            if (table == null) {
+                // tableId对应的记录不存在
+                throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
+            }
+
+            String fullname = table.getDbName() + "." + table.getTableName();
+            // check name filter
+            if (nameFilter != null && !nameFilter.filter(fullname)) {
+                return null;
+            }
+            if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
+                return null;
+            }
+
+            EventType eventType = null;
+            int type = event.getHeader().getType();
+            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
+                eventType = EventType.INSERT;
+            } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
+                eventType = EventType.UPDATE;
+            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
+                eventType = EventType.DELETE;
+            } else {
+                throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
+            }
+
+            Header header = createHeader(binlogFileName,
+                event.getHeader(),
+                table.getDbName(),
+                table.getTableName(),
+                eventType);
+            RowChange.Builder rowChangeBuider = RowChange.newBuilder();
+            rowChangeBuider.setTableId(event.getTableId());
+            rowChangeBuider.setIsDdl(false);
+
+            rowChangeBuider.setEventType(eventType);
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+            BitSet columns = event.getColumns();
+            BitSet changeColumns = event.getColumns();
+            boolean tableError = false;
+            TableMeta tableMeta = null;
+            if (tableMetaCache != null) {// 入错存在table meta cache
+                tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
+                if (tableMeta == null) {
+                    tableError = true;
+                    if (!filterTableError) {
+                        throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
+                    }
+                }
+            }
+
+            while (buffer.nextOneRow(columns)) {
+                // 处理row记录
+                RowData.Builder rowDataBuilder = RowData.newBuilder();
+                if (EventType.INSERT == eventType) {
+                    // insert的记录放在before字段中
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
+                } else if (EventType.DELETE == eventType) {
+                    // delete的记录放在before字段中
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
+                } else {
+                    // update需要处理before/after
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
+                    if (!buffer.nextOneRow(changeColumns)) {
+                        rowChangeBuider.addRowDatas(rowDataBuilder.build());
+                        break;
+                    }
+
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, event.getChangeColumns(), true, tableMeta);
+                }
+
+                rowChangeBuider.addRowDatas(rowDataBuilder.build());
+            }
+
+            RowChange rowChange = rowChangeBuider.build();
+            if (tableError) {
+                Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
+                logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
+                return null;
+            } else {
+                Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
+                return entry;
+            }
+        } catch (Exception e) {
+            throw new CanalParseException("parse row data failed.", e);
+        }
+    }
+
+    private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols,
+                                boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
+        final int columnCnt = event.getTable().getColumnCnt();
+        final ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
+
+        boolean tableError = false;
+        // check table fileds count，只能处理加字段
+        if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
+            // online ddl增加字段操作步骤：
+            // 1. 新增一张临时表，将需要做ddl表的数据全量导入
+            // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
+            // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作
+            // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作
+            // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作
+            tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false);// 强制重新获取一次
+            if (tableMeta == null) {
+                tableError = true;
+                if (!filterTableError) {
+                    throw new CanalParseException("not found [" + event.getTable().getDbName() + "."
+                                                  + event.getTable().getTableName() + "] in db , pls check!");
+                }
+            }
+
+            // 在做一次判断
+            if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
+                tableError = true;
+                if (!filterTableError) {
+                    throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + ","
+                                                  + columnInfo.length + " vs " + tableMeta.getFileds().size());
+                }
+            }
+        }
+
+        for (int i = 0; i < columnCnt; i++) {
+            ColumnInfo info = columnInfo[i];
+            Column.Builder columnBuilder = Column.newBuilder();
+
+            FieldMeta fieldMeta = null;
+            if (tableMeta != null && !tableError) {
+                // 处理file meta
+                fieldMeta = tableMeta.getFileds().get(i);
+                columnBuilder.setName(fieldMeta.getColumnName());
+                columnBuilder.setIsKey(fieldMeta.isKey());
+                // 增加mysql type类型,issue 73
+                columnBuilder.setMysqlType(fieldMeta.getColumnType());
+            }
+            columnBuilder.setIndex(i);
+            columnBuilder.setIsNull(false);
+
+            // fixed issue
+            // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
+            boolean isBinary = false;
+            if (fieldMeta != null) {
+                if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
+                    isBinary = true;
+                } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
+                    isBinary = true;
+                }
+            }
+            buffer.nextValue(info.type, info.meta, isBinary);
+
+            int javaType = buffer.getJavaType();
+            if (buffer.isNull()) {
+                columnBuilder.setIsNull(true);
+            } else {
+                final Serializable value = buffer.getValue();
+                // 处理各种类型
+                switch (javaType) {
+                    case Types.INTEGER:
+                    case Types.TINYINT:
+                    case Types.SMALLINT:
+                    case Types.BIGINT:
+                        // 处理unsigned类型
+                        Number number = (Number) value;
+                        if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
+                            switch (buffer.getLength()) {
+                                case 1: /* MYSQL_TYPE_TINY */
+                                    columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE
+                                                                                          + number.intValue())));
+                                    javaType = Types.SMALLINT; // 往上加一个量级
+                                    break;
+
+                                case 2: /* MYSQL_TYPE_SHORT */
+                                    columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE
+                                                                                          + number.intValue())));
+                                    javaType = Types.INTEGER; // 往上加一个量级
+                                    break;
+
+                                case 3: /* MYSQL_TYPE_INT24 */
+                                    columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE
+                                                                                          + number.intValue())));
+                                    javaType = Types.INTEGER; // 往上加一个量级
+                                    break;
+
+                                case 4: /* MYSQL_TYPE_LONG */
+                                    columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE
+                                                                                       + number.longValue())));
+                                    javaType = Types.BIGINT; // 往上加一个量级
+                                    break;
+
+                                case 8: /* MYSQL_TYPE_LONGLONG */
+                                    columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue()))
+                                        .toString());
+                                    javaType = Types.DECIMAL; // 往上加一个量级，避免执行出错
+                                    break;
+                            }
+                        } else {
+                            // 对象为number类型，直接valueof即可
+                            columnBuilder.setValue(String.valueOf(value));
+                        }
+                        break;
+                    case Types.REAL: // float
+                    case Types.DOUBLE: // double
+                        // 对象为number类型，直接valueof即可
+                        columnBuilder.setValue(String.valueOf(value));
+                        break;
+                    case Types.BIT:// bit
+                        // 对象为number类型
+                        columnBuilder.setValue(String.valueOf(value));
+                        break;
+                    case Types.DECIMAL:
+                        columnBuilder.setValue(((BigDecimal) value).toPlainString());
+                        break;
+                    case Types.TIMESTAMP:
+                        // 修复时间边界值
+                        // String v = value.toString();
+                        // v = v.substring(0, v.length() - 2);
+                        // columnBuilder.setValue(v);
+                        // break;
+                    case Types.TIME:
+                    case Types.DATE:
+                        // 需要处理year
+                        columnBuilder.setValue(value.toString());
+                        break;
+                    case Types.BINARY:
+                    case Types.VARBINARY:
+                    case Types.LONGVARBINARY:
+                        // fixed text encoding
+                        // https://github.com/AlibabaTech/canal/issues/18
+                        // mysql binlog中blob/text都处理为blob类型，需要反查table
+                        // meta，按编码解析text
+                        if (fieldMeta != null && isText(fieldMeta.getColumnType())) {
+                            columnBuilder.setValue(new String((byte[]) value, charset));
+                            javaType = Types.CLOB;
+                        } else {
+                            // byte数组，直接使用iso-8859-1保留对应编码，浪费内存
+                            columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
+                            javaType = Types.BLOB;
+                        }
+                        break;
+                    case Types.CHAR:
+                    case Types.VARCHAR:
+                        columnBuilder.setValue(value.toString());
+                        break;
+                    default:
+                        columnBuilder.setValue(value.toString());
+                }
+
+            }
+
+            columnBuilder.setSqlType(javaType);
+            // 设置是否update的标记位
+            columnBuilder.setUpdated(isAfter
+                                     && isUpdate(rowDataBuilder.getBeforeColumnsList(),
+                                         columnBuilder.getIsNull() ? null : columnBuilder.getValue(),
+                                         i));
+            if (isAfter) {
+                rowDataBuilder.addAfterColumns(columnBuilder.build());
+            } else {
+                rowDataBuilder.addBeforeColumns(columnBuilder.build());
+            }
+        }
+
+        return tableError;
+
+    }
+
+    private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
+        Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
+        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
+        rowChangeBuider.setSql(queryString);
+        rowChangeBuider.setEventType(EventType.QUERY);
+        return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
+    }
+
+    private Header createHeader(String binlogFile, LogHeader logHeader, String schemaName, String tableName,
+                                EventType eventType) {
+        // header会做信息冗余,方便以后做检索或者过滤
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setVersion(version);
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
+        headerBuilder.setServerId(logHeader.getServerId());
+        headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
+        headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L);
+        headerBuilder.setSourceType(Type.MYSQL);
+        if (eventType != null) {
+            headerBuilder.setEventType(eventType);
+        }
+        if (schemaName != null) {
+            headerBuilder.setSchemaName(schemaName);
+        }
+        if (tableName != null) {
+            headerBuilder.setTableName(tableName);
+        }
+        headerBuilder.setEventLength(logHeader.getEventLen());
+        return headerBuilder.build();
+    }
+
+    private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
+        if (bfColumns == null) {
+            throw new CanalParseException("ERROR ## the bfColumns is null");
+        }
+
+        if (index < 0) {
+            return false;
+        }
+        if ((bfColumns.size() - 1) < index) {
+            return false;
+        }
+        Column column = bfColumns.get(index);
+
+        if (column.getIsNull()) {
+            if (newValue != null) {
+                return true;
+            }
+        } else {
+            if (newValue == null) {
+                return true;
+            } else {
+                if (!column.getValue().equals(newValue)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
+        try {
+            return tableMetaCache.getTableMeta(dbName, tbName, useCache);
+        } catch (Exception e) {
+            String message = ExceptionUtils.getRootCauseMessage(e);
+            if (filterTableError) {
+                if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
+                    return null;
+                }
+            }
+
+            throw new CanalParseException(e);
+        }
+    }
+
+    private boolean isText(String columnType) {
+        return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
+               || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
+    }
+
+    public static TransactionBegin createTransactionBegin(long threadId) {
+        TransactionBegin.Builder beginBuilder = TransactionBegin.newBuilder();
+        beginBuilder.setThreadId(threadId);
+        return beginBuilder.build();
+    }
+
+    public static TransactionEnd createTransactionEnd(long transactionId) {
+        TransactionEnd.Builder endBuilder = TransactionEnd.newBuilder();
+        endBuilder.setTransactionId(String.valueOf(transactionId));
+        return endBuilder.build();
+    }
+
+    public static Pair createSpecialPair(String key, String value) {
+        Pair.Builder pairBuilder = Pair.newBuilder();
+        pairBuilder.setKey(key);
+        pairBuilder.setValue(value);
+        return pairBuilder.build();
+    }
+
+    public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(header);
+        entryBuilder.setEntryType(entryType);
+        entryBuilder.setStoreValue(storeValue);
+        return entryBuilder.build();
+    }
+
+    public void setCharset(Charset charset) {
+        this.charset = charset;
+    }
+
+    public void setNameFilter(AviaterRegexFilter nameFilter) {
+        this.nameFilter = nameFilter;
+    }
+
+    public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
+        this.nameBlackFilter = nameBlackFilter;
+    }
+
+    public void setTableMetaCache(TableMetaCache tableMetaCache) {
+        this.tableMetaCache = tableMetaCache;
+    }
+
+    public void setFilterQueryDcl(boolean filterQueryDcl) {
+        this.filterQueryDcl = filterQueryDcl;
+    }
+
+    public void setFilterQueryDml(boolean filterQueryDml) {
+        this.filterQueryDml = filterQueryDml;
+    }
+
+    public void setFilterQueryDdl(boolean filterQueryDdl) {
+        this.filterQueryDdl = filterQueryDdl;
+    }
+
+    public void setFilterTableError(boolean filterTableError) {
+        this.filterTableError = filterTableError;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java

@@ -0,0 +1,275 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.oro.text.regex.Perl5Matcher;
+
+import com.alibaba.otter.canal.filter.PatternUtils;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+
+/**
+ * 简单的ddl解析工具类，后续可使用cobar/druid的SqlParser进行语法树解析
+ * 
+ * <pre>
+ * 解析支持：
+ * a. 带schema: retl.retl_mark
+ * b. 带引号` :  `retl.retl_mark`
+ * c. 存在换行符： create table \n `retl.retl_mark`
+ * </pre>
+ * 
+ * http://dev.mysql.com/doc/refman/5.6/en/sql-syntax-data-definition.html
+ * 
+ * @author jianghang 2013-1-22 下午10:03:22
+ * @version 1.0.0
+ */
+public class SimpleDdlParser {
+
+    public static final String CREATE_PATTERN       = "^\\s*CREATE\\s*(TEMPORARY)?\\s*TABLE\\s*(.*)$";
+    public static final String DROP_PATTERN         = "^\\s*DROP\\s*(TEMPORARY)?\\s*TABLE\\s*(.*)$";
+    public static final String ALERT_PATTERN        = "^\\s*ALTER\\s*(IGNORE)?\\s*TABLE\\s*(.*)$";
+    public static final String TRUNCATE_PATTERN     = "^\\s*TRUNCATE\\s*(TABLE)?\\s*(.*)$";
+    public static final String TABLE_PATTERN        = "^(IF\\s*NOT\\s*EXISTS\\s*)?(IF\\s*EXISTS\\s*)?(`?.+?`?[;\\(\\s]+?)?.*$"; // 采用非贪婪模式
+    public static final String INSERT_PATTERN       = "^\\s*(INSERT|MERGE|REPLACE)(.*)$";
+    public static final String UPDATE_PATTERN       = "^\\s*UPDATE(.*)$";
+    public static final String DELETE_PATTERN       = "^\\s*DELETE(.*)$";
+    public static final String RENAME_PATTERN       = "^\\s*RENAME\\s*TABLE\\s*(.*?)\\s*TO\\s*(.*?)$";
+    /**
+     * <pre>
+     * CREATE [UNIQUE|FULLTEXT|SPATIAL] INDEX index_name
+     *         [index_type]
+     *         ON tbl_name (index_col_name,...)
+     *         [algorithm_option | lock_option] ...
+     *         
+     * http://dev.mysql.com/doc/refman/5.6/en/create-index.html
+     * </pre>
+     */
+    public static final String CREATE_INDEX_PATTERN = "^\\s*CREATE\\s*.*?\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
+    public static final String DROP_INDEX_PATTERN   = "^\\s*DROP\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
+
+    public static DdlResult parse(String queryString, String schmeaName) {
+        queryString = removeComment(queryString); // 去除/* */的sql注释内容
+        DdlResult result = parseDdl(queryString, schmeaName, ALERT_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.ALTER);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, CREATE_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.CREATE);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, DROP_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.ERASE);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, TRUNCATE_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.TRUNCATE);
+            return result;
+        }
+
+        result = parseRename(queryString, schmeaName, RENAME_PATTERN);
+        if (result != null) {
+            result.setType(EventType.RENAME);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, CREATE_INDEX_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.CINDEX);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, DROP_INDEX_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.DINDEX);
+            return result;
+        }
+
+        result = new DdlResult(schmeaName);
+        if (isDml(queryString, INSERT_PATTERN)) {
+            result.setType(EventType.INSERT);
+            return result;
+        }
+
+        if (isDml(queryString, UPDATE_PATTERN)) {
+            result.setType(EventType.UPDATE);
+            return result;
+        }
+
+        if (isDml(queryString, DELETE_PATTERN)) {
+            result.setType(EventType.DELETE);
+            return result;
+        }
+
+        result.setType(EventType.QUERY);
+        return result;
+    }
+
+    private static DdlResult parseDdl(String queryString, String schmeaName, String pattern, int index) {
+        Perl5Matcher matcher = new Perl5Matcher();
+        if (matcher.matches(queryString, PatternUtils.getPattern(pattern))) {
+            DdlResult result = parseTableName(matcher.getMatch().group(index), schmeaName);
+            return result != null ? result : new DdlResult(schmeaName); // 无法解析时，直接返回schmea，进行兼容处理
+        }
+
+        return null;
+    }
+
+    private static boolean isDml(String queryString, String pattern) {
+        Perl5Matcher matcher = new Perl5Matcher();
+        if (matcher.matches(queryString, PatternUtils.getPattern(pattern))) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private static DdlResult parseRename(String queryString, String schmeaName, String pattern) {
+        Perl5Matcher matcher = new Perl5Matcher();
+        if (matcher.matches(queryString, PatternUtils.getPattern(pattern))) {
+            DdlResult orign = parseTableName(matcher.getMatch().group(1), schmeaName);
+            DdlResult target = parseTableName(matcher.getMatch().group(2), schmeaName);
+            if (orign != null && target != null) {
+                return new DdlResult(target.getSchemaName(),
+                    target.getTableName(),
+                    orign.getSchemaName(),
+                    orign.getTableName());
+            }
+        }
+
+        return null;
+    }
+
+    private static DdlResult parseTableName(String matchString, String schmeaName) {
+        Perl5Matcher tableMatcher = new Perl5Matcher();
+        matchString = matchString + " ";
+        if (tableMatcher.matches(matchString, PatternUtils.getPattern(TABLE_PATTERN))) {
+            String tableString = tableMatcher.getMatch().group(3);
+
+            tableString = StringUtils.removeEnd(tableString, ";");
+            tableString = StringUtils.removeEnd(tableString, "(");
+            tableString = StringUtils.trim(tableString);
+            // 特殊处理引号`
+            tableString = removeEscape(tableString);
+            // 处理schema.table的写法
+            String names[] = StringUtils.split(tableString, ".");
+            if (names != null && names.length > 1) {
+                return new DdlResult(removeEscape(names[0]), removeEscape(names[1]));
+            } else {
+                return new DdlResult(schmeaName, removeEscape(names[0]));
+            }
+        }
+
+        return null;
+    }
+
+    private static String removeEscape(String str) {
+        String result = StringUtils.removeEnd(str, "`");
+        result = StringUtils.removeStart(result, "`");
+        return result;
+    }
+
+    private static String removeComment(String sql) {
+        if (sql == null) {
+            return null;
+        }
+
+        String start = "/*";
+        String end = "*/";
+        while (true) {
+            // 循环找到所有的注释
+            int index0 = sql.indexOf(start);
+            if (index0 == -1) {
+                return sql;
+            }
+            int index1 = sql.indexOf(end, index0);
+            if (index1 == -1) {
+                return sql;
+            }
+            StringBuilder sb = new StringBuilder();
+            sb.append(sql.substring(0, index0));
+            sb.append(" ");
+            sb.append(sql.substring(index1 + end.length()));
+            sql = sb.toString();
+        }
+    }
+
+    public static class DdlResult {
+
+        private String    schemaName;
+        private String    tableName;
+        private String    oriSchemaName; // rename ddl中的源表
+        private String    oriTableName; // rename ddl中的目标表
+        private EventType type;
+
+        public DdlResult(){
+        }
+
+        public DdlResult(String schemaName){
+            this.schemaName = schemaName;
+        }
+
+        public DdlResult(String schemaName, String tableName){
+            this.schemaName = schemaName;
+            this.tableName = tableName;
+        }
+
+        public DdlResult(String schemaName, String tableName, String oriSchemaName, String oriTableName){
+            this.schemaName = schemaName;
+            this.tableName = tableName;
+            this.oriSchemaName = oriSchemaName;
+            this.oriTableName = oriTableName;
+        }
+
+        public String getSchemaName() {
+            return schemaName;
+        }
+
+        public void setSchemaName(String schemaName) {
+            this.schemaName = schemaName;
+        }
+
+        public String getTableName() {
+            return tableName;
+        }
+
+        public void setTableName(String tableName) {
+            this.tableName = tableName;
+        }
+
+        public EventType getType() {
+            return type;
+        }
+
+        public void setType(EventType type) {
+            this.type = type;
+        }
+
+        public String getOriSchemaName() {
+            return oriSchemaName;
+        }
+
+        public void setOriSchemaName(String oriSchemaName) {
+            this.oriSchemaName = oriSchemaName;
+        }
+
+        public String getOriTableName() {
+            return oriTableName;
+        }
+
+        public void setOriTableName(String oriTableName) {
+            this.oriTableName = oriTableName;
+        }
+
+        @Override
+        public String toString() {
+            return "DdlResult [schemaName=" + schemaName + ", tableName=" + tableName + ", oriSchemaName="
+                   + oriSchemaName + ", oriTableName=" + oriTableName + ", type=" + type + "]";
+        }
+
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java

@@ -0,0 +1,139 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 处理table meta解析和缓存
+ * 
+ * @author jianghang 2013-1-17 下午10:15:16
+ * @version 1.0.0
+ */
+public class TableMetaCache {
+
+    public static final String     COLUMN_NAME    = "COLUMN_NAME";
+    public static final String     COLUMN_TYPE    = "COLUMN_TYPE";
+    public static final String     IS_NULLABLE    = "IS_NULLABLE";
+    public static final String     COLUMN_KEY     = "COLUMN_KEY";
+    public static final String     COLUMN_DEFAULT = "COLUMN_DEFAULT";
+    public static final String     EXTRA          = "EXTRA";
+    private MysqlConnection        connection;
+
+    // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表
+    private Map<String, TableMeta> tableMetaCache;
+
+    public TableMetaCache(MysqlConnection con){
+        this.connection = con;
+        tableMetaCache = new MapMaker().makeComputingMap(new Function<String, TableMeta>() {
+
+            public TableMeta apply(String name) {
+                try {
+                    return getTableMeta0(name);
+                } catch (IOException e) {
+                    // 尝试做一次retry操作
+                    try {
+                        connection.reconnect();
+                        return getTableMeta0(name);
+                    } catch (IOException e1) {
+                        throw new CanalParseException("fetch failed by table meta:" + name, e1);
+                    }
+                }
+            }
+
+        });
+
+    }
+
+    public TableMeta getTableMeta(String schema, String table) {
+        return getTableMeta(schema, table, true);
+    }
+
+    public TableMeta getTableMeta(String schema, String table, boolean useCache) {
+        if (!useCache) {
+            tableMetaCache.remove(getFullName(schema, table));
+        }
+
+        return tableMetaCache.get(getFullName(schema, table));
+    }
+
+    public void clearTableMeta(String schema, String table) {
+        tableMetaCache.remove(getFullName(schema, table));
+    }
+
+    public void clearTableMetaWithSchemaName(String schema) {
+        // Set<String> removeNames = new HashSet<String>(); //
+        // 存一份临时变量，避免在遍历的时候进行删除
+        for (String name : tableMetaCache.keySet()) {
+            if (StringUtils.startsWithIgnoreCase(name, schema + ".")) {
+                // removeNames.add(name);
+                tableMetaCache.remove(name);
+            }
+        }
+
+        // for (String name : removeNames) {
+        // tables.remove(name);
+        // }
+    }
+
+    public void clearTableMeta() {
+        tableMetaCache.clear();
+    }
+
+    private TableMeta getTableMeta0(String fullname) throws IOException {
+        ResultSetPacket packet = connection.query("desc " + fullname);
+        return new TableMeta(fullname, parserTableMeta(packet));
+    }
+
+    private List<FieldMeta> parserTableMeta(ResultSetPacket packet) {
+        Map<String, Integer> nameMaps = new HashMap<String, Integer>(6, 1f);
+
+        int index = 0;
+        for (FieldPacket fieldPacket : packet.getFieldDescriptors()) {
+            nameMaps.put(fieldPacket.getOriginalName(), index++);
+        }
+
+        int size = packet.getFieldDescriptors().size();
+        int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size();
+        List<FieldMeta> result = new ArrayList<FieldMeta>();
+        for (int i = 0; i < count; i++) {
+            FieldMeta meta = new FieldMeta();
+            // 做一个优化，使用String.intern()，共享String对象，减少内存使用
+            meta.setColumnName(packet.getFieldValues().get(nameMaps.get(COLUMN_NAME) + i * size).intern());
+            meta.setColumnType(packet.getFieldValues().get(nameMaps.get(COLUMN_TYPE) + i * size));
+            meta.setIsNullable(packet.getFieldValues().get(nameMaps.get(IS_NULLABLE) + i * size));
+            meta.setIskey(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size));
+            meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size));
+            meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size));
+
+            result.add(meta);
+        }
+
+        return result;
+    }
+
+    private String getFullName(String schema, String table) {
+        StringBuilder builder = new StringBuilder();
+        return builder.append('`')
+            .append(schema)
+            .append('`')
+            .append('.')
+            .append('`')
+            .append(table)
+            .append('`')
+            .toString();
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java

@@ -0,0 +1,225 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.local;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+
+/**
+ * 维护binlog文件列表
+ * 
+ * @author jianghang 2012-7-7 下午03:48:05
+ * @version 1.0.0
+ */
+public class BinLogFileQueue {
+
+    private String        baseName       = "mysql-bin.";
+    private List<File>    binlogs        = new ArrayList<File>();
+    private File          directory;
+    private ReentrantLock lock           = new ReentrantLock();
+    private Condition     nextCondition  = lock.newCondition();
+    private Timer         timer          = new Timer(true);
+    private long          reloadInterval = 10 * 1000L;           // 10秒
+
+    public BinLogFileQueue(String directory){
+        this(new File(directory));
+    }
+
+    public BinLogFileQueue(File directory){
+        this.directory = directory;
+
+        if (!directory.canRead()) {
+            throw new CanalParseException("Binlog index missing or unreadable;  " + directory.getAbsolutePath());
+        }
+
+        List<File> files = listBinlogFiles();
+        for (File file : files) {
+            offer(file);
+        }
+
+        timer.scheduleAtFixedRate(new TimerTask() {
+
+            public void run() {
+                List<File> files = listBinlogFiles();
+                for (File file : files) {
+                    offer(file);
+                }
+            }
+        }, reloadInterval, reloadInterval);
+    }
+
+    /**
+     * 根据前一个文件，获取符合条件的下一个binlog文件
+     * 
+     * @param pre
+     * @return
+     */
+    public File getNextFile(File pre) {
+        try {
+            lock.lockInterruptibly();
+            if (binlogs.size() == 0) {
+                return null;
+            } else {
+                if (pre == null) {// 第一次
+                    return binlogs.get(0);
+                } else {
+                    int index = seek(pre);
+                    if (index < binlogs.size() - 1) {
+                        return binlogs.get(index + 1);
+                    } else {
+                        return null;
+                    }
+                }
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return null;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public File getBefore(File file) {
+        try {
+            lock.lockInterruptibly();
+            if (binlogs.size() == 0) {
+                return null;
+            } else {
+                if (file == null) {// 第一次
+                    return binlogs.get(binlogs.size() - 1);
+                } else {
+                    int index = seek(file);
+                    if (index > 0) {
+                        return binlogs.get(index - 1);
+                    } else {
+                        return null;
+                    }
+                }
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return null;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * 根据前一个文件，获取符合条件的下一个binlog文件
+     * 
+     * @param pre
+     * @return
+     * @throws InterruptedException
+     */
+    public File waitForNextFile(File pre) throws InterruptedException {
+        try {
+            lock.lockInterruptibly();
+            if (binlogs.size() == 0) {
+                nextCondition.await();// 等待新文件
+            }
+
+            if (pre == null) {// 第一次
+                return binlogs.get(0);
+            } else {
+                int index = seek(pre);
+                if (index < binlogs.size() - 1) {
+                    return binlogs.get(index + 1);
+                } else {
+                    nextCondition.await();// 等待新文件
+                    return waitForNextFile(pre);// 唤醒之后递归调用一下
+                }
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * 获取当前所有binlog文件
+     */
+    public List<File> currentBinlogs() {
+        return new ArrayList<File>(binlogs);
+    }
+
+    public void destory() {
+        try {
+            lock.lockInterruptibly();
+            timer.cancel();
+            binlogs.clear();
+
+            nextCondition.signalAll();// 唤醒线程，通知退出
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private boolean offer(File file) {
+        try {
+            lock.lockInterruptibly();
+            if (!binlogs.contains(file)) {
+                binlogs.add(file);
+                nextCondition.signalAll();// 唤醒
+                return true;
+            } else {
+                return false;
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return false;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private List<File> listBinlogFiles() {
+        List<File> files = new ArrayList<File>();
+        files.addAll(FileUtils.listFiles(directory, new IOFileFilter() {
+
+            public boolean accept(File file) {
+                return file.getName().startsWith(baseName);
+            }
+
+            public boolean accept(File dir, String name) {
+                return true;
+            }
+        }, null));
+        // 排一下序列
+        Collections.sort(files, new Comparator<File>() {
+
+            public int compare(File o1, File o2) {
+                return o1.getName().compareTo(o2.getName());
+            }
+
+        });
+        return files;
+    }
+
+    private int seek(File file) {
+        for (int i = 0; i < binlogs.size(); i++) {
+            File binlog = binlogs.get(i);
+            if (binlog.getName().equals(file.getName())) {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    // ================== setter / getter ===================
+
+    public void setBaseName(String baseName) {
+        this.baseName = baseName;
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BufferedFileDataInput.java

@@ -0,0 +1,92 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.local;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.FileChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author jianghang 2012-7-7 下午03:10:47
+ * @version 1.0.0
+ */
+public class BufferedFileDataInput {
+
+    private static final Logger logger = LoggerFactory.getLogger(BufferedFileDataInput.class);
+    // Read parameters.
+    private File                file;
+    private int                 size;
+
+    // Variables to control reading.
+    private FileInputStream     fileInput;
+    private BufferedInputStream bufferedInput;
+    private DataInputStream     dataInput;
+    private long                offset;
+    private FileChannel         fileChannel;
+
+    public BufferedFileDataInput(File file, int size) throws FileNotFoundException, IOException, InterruptedException{
+        this.file = file;
+        this.size = size;
+    }
+
+    public BufferedFileDataInput(File file) throws FileNotFoundException, IOException, InterruptedException{
+        this(file, 1024);
+    }
+
+    public long available() throws IOException {
+        return fileChannel.size() - offset;
+    }
+
+    public long skip(long bytes) throws IOException {
+        long bytesSkipped = bufferedInput.skip(bytes);
+        offset += bytesSkipped;
+        return bytesSkipped;
+    }
+
+    public void seek(long seekBytes) throws FileNotFoundException, IOException, InterruptedException {
+        fileInput = new FileInputStream(file);
+        fileChannel = fileInput.getChannel();
+
+        try {
+            fileChannel.position(seekBytes);
+        } catch (ClosedByInterruptException e) {
+            throw new InterruptedException();
+        }
+        bufferedInput = new BufferedInputStream(fileInput, size);
+        dataInput = new DataInputStream(bufferedInput);
+        offset = seekBytes;
+    }
+
+    public void readFully(byte[] bytes) throws IOException {
+        readFully(bytes, 0, bytes.length);
+    }
+
+    public void readFully(byte[] bytes, int start, int len) throws IOException {
+        dataInput.readFully(bytes, start, len);
+        offset += len;
+    }
+
+    public void close() {
+        try {
+            if (fileChannel != null) {
+                fileChannel.close();
+                fileInput.close();
+            }
+        } catch (IOException e) {
+            logger.warn("Unable to close buffered file reader: file=" + file.getName() + " exception=" + e.getMessage());
+        }
+
+        fileChannel = null;
+        fileInput = null;
+        bufferedInput = null;
+        dataInput = null;
+        offset = -1;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java

@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.parse.index;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 接口组合
+ * 
+ * @author jianghang 2012-7-7 上午10:02:02
+ * @version 1.0.0
+ */
+public interface CanalLogPositionManager extends CanalLifeCycle {
+
+    LogPosition getLatestIndexBy(String destination);
+
+    void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException;
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/FailbackLogPositionManager.java

@@ -0,0 +1,74 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 实现基于failover查找的机制完成meta的操作
+ * 
+ * <pre>
+ * 应用场景：比如针对内存buffer，出现HA切换，先尝试从内存buffer区中找到lastest position，如果不存在才尝试找一下meta里消费的信息
+ * </pre>
+ * 
+ * @author jianghang 2012-7-20 下午02:33:20
+ */
+public class FailbackLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    private CanalLogPositionManager primary;
+    private CanalLogPositionManager failback;
+
+    public void start() {
+        super.start();
+        Assert.notNull(primary);
+        Assert.notNull(failback);
+
+        if (!primary.isStart()) {
+            primary.start();
+        }
+
+        if (!failback.isStart()) {
+            failback.start();
+        }
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (primary.isStart()) {
+            primary.stop();
+        }
+
+        if (failback.isStart()) {
+            failback.stop();
+        }
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPosition = primary.getLatestIndexBy(destination);
+        if (logPosition == null) {
+            return failback.getLatestIndexBy(destination);
+        } else {
+            return logPosition;
+        }
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
+        try {
+            primary.persistLogPosition(destination, logPosition);
+        } catch (CanalParseException e) {
+            failback.persistLogPosition(destination, logPosition);
+        }
+    }
+
+    public void setPrimary(CanalLogPositionManager primary) {
+        this.primary = primary;
+    }
+
+    public void setFailback(CanalLogPositionManager failback) {
+        this.failback = failback;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java

@@ -0,0 +1,194 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于文件刷新的log position实现
+ * 
+ * <pre>
+ * 策略：
+ * 1. 先写内存，然后定时刷新数据到File
+ * 2. 数据采取overwrite模式(只保留最后一次)
+ * </pre>
+ * 
+ * @author jianghang 2013-4-15 下午09:40:48
+ * @version 1.0.4
+ */
+public class FileMixedLogPositionManager extends MemoryLogPositionManager {
+
+    private static final Logger      logger       = LoggerFactory.getLogger(FileMixedLogPositionManager.class);
+    private static final Charset     charset      = Charset.forName("UTF-8");
+    private File                     dataDir;
+    private String                   dataFileName = "parse.dat";
+    private Map<String, File>        dataFileCaches;
+    private ScheduledExecutorService executor;
+    @SuppressWarnings("serial")
+    private final LogPosition        nullPosition = new LogPosition() {
+                                                  };
+
+    private long                     period       = 1000;                                                      // 单位ms
+    private Set<String>              persistTasks;
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(dataDir);
+        if (!dataDir.exists()) {
+            try {
+                FileUtils.forceMkdir(dataDir);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+
+        if (!dataDir.canRead() || !dataDir.canWrite()) {
+            throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
+        }
+
+        dataFileCaches = new MapMaker().makeComputingMap(new Function<String, File>() {
+
+            public File apply(String destination) {
+                return getDataFile(destination);
+            }
+        });
+
+        executor = Executors.newScheduledThreadPool(1);
+        positions = new MapMaker().makeComputingMap(new Function<String, LogPosition>() {
+
+            public LogPosition apply(String destination) {
+                LogPosition logPosition = loadDataFromFile(dataFileCaches.get(destination));
+                if (logPosition == null) {
+                    return nullPosition;
+                } else {
+                    return logPosition;
+                }
+            }
+        });
+
+        persistTasks = Collections.synchronizedSet(new HashSet<String>());
+
+        // 启动定时工作任务
+        executor.scheduleAtFixedRate(new Runnable() {
+
+            public void run() {
+                List<String> tasks = new ArrayList<String>(persistTasks);
+                for (String destination : tasks) {
+                    try {
+                        // 定时将内存中的最新值刷到file中，多次变更只刷一次
+                        flushDataToFile(destination);
+                        persistTasks.remove(destination);
+                    } catch (Throwable e) {
+                        // ignore
+                        logger.error("period update" + destination + " curosr failed!", e);
+                    }
+                }
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    public void stop() {
+        super.stop();
+
+        flushDataToFile();
+        executor.shutdownNow();
+        positions.clear();
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        persistTasks.add(destination);// 添加到任务队列中进行触发
+        super.persistLogPosition(destination, logPosition);
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPostion = super.getLatestIndexBy(destination);
+        if (logPostion == nullPosition) {
+            return null;
+        } else {
+            return logPostion;
+        }
+    }
+
+    // ============================ helper method ======================
+
+    private File getDataFile(String destination) {
+        File destinationMetaDir = new File(dataDir, destination);
+        if (!destinationMetaDir.exists()) {
+            try {
+                FileUtils.forceMkdir(destinationMetaDir);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+
+        return new File(destinationMetaDir, dataFileName);
+    }
+
+    private void flushDataToFile() {
+        for (String destination : positions.keySet()) {
+            flushDataToFile(destination);
+        }
+    }
+
+    private void flushDataToFile(String destination) {
+        flushDataToFile(destination, dataFileCaches.get(destination));
+    }
+
+    private void flushDataToFile(String destination, File dataFile) {
+        LogPosition position = positions.get(destination);
+        if (position != null && position != nullPosition) {
+            String json = JsonUtils.marshalToString(position);
+            try {
+                FileUtils.writeStringToFile(dataFile, json);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+    }
+
+    private LogPosition loadDataFromFile(File dataFile) {
+        try {
+            if (!dataFile.exists()) {
+                return null;
+            }
+
+            String json = FileUtils.readFileToString(dataFile, charset.name());
+            return JsonUtils.unmarshalFromString(json, LogPosition.class);
+        } catch (IOException e) {
+            throw new CanalMetaManagerException(e);
+        }
+    }
+
+    public void setDataDir(String dataDir) {
+        this.dataDir = new File(dataDir);
+    }
+
+    public void setDataDir(File dataDir) {
+        this.dataDir = dataDir;
+    }
+
+    public void setPeriod(long period) {
+        this.period = period;
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java

@@ -0,0 +1,39 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.Map;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于内存的实现
+ * 
+ * @author jianghang 2012-7-7 上午10:17:23
+ * @version 1.0.0
+ */
+public class MemoryLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    protected Map<String, LogPosition> positions;
+
+    public void start() {
+        super.start();
+
+        positions = new MapMaker().makeMap();
+    }
+
+    public void stop() {
+        super.stop();
+
+        positions.clear();
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        return positions.get(destination);
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        positions.put(destination, logPosition);
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManager.java

@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.store.helper.CanalEventUtils;
+
+/**
+ * 基于meta信息的实现
+ * 
+ * @author jianghang 2012-7-10 下午05:02:33
+ * @version 1.0.0
+ */
+public class MetaLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    private static final Logger logger = LoggerFactory.getLogger(MetaLogPositionManager.class);
+    private CanalMetaManager    metaManager;
+
+    public void start() {
+        super.start();
+        Assert.notNull(metaManager);
+        if (!metaManager.isStart()) {
+            metaManager.start();
+        }
+    }
+
+    public void stop() {
+        super.stop();
+        if (metaManager.isStart()) {
+            metaManager.stop();
+        }
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        // do nothing
+        logger.info("persist LogPosition:{}", destination, logPosition);
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        List<ClientIdentity> clientIdentitys = metaManager.listAllSubscribeInfo(destination);
+        LogPosition result = null;
+        if (!CollectionUtils.isEmpty(clientIdentitys)) {
+            // 尝试找到一个最小的logPosition
+            for (ClientIdentity clientIdentity : clientIdentitys) {
+                LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
+                if (position == null) {
+                    continue;
+                }
+
+                if (result == null) {
+                    result = position;
+                } else {
+                    result = CanalEventUtils.min(result, position);
+                }
+            }
+        }
+
+        return result;
+    }
+
+    public void setMetaManager(CanalMetaManager metaManager) {
+        this.metaManager = metaManager;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java

@@ -0,0 +1,90 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 混合memory + zookeeper的存储模式
+ * 
+ * @author jianghang 2012-7-7 上午10:33:19
+ * @version 1.0.0
+ */
+public class MixedLogPositionManager extends MemoryLogPositionManager implements CanalLogPositionManager {
+
+    private static final Logger         logger       = LoggerFactory.getLogger(MixedLogPositionManager.class);
+    private ZooKeeperLogPositionManager zooKeeperLogPositionManager;
+    private ExecutorService             executor;
+    @SuppressWarnings("serial")
+    private final LogPosition           nullPosition = new LogPosition() {
+                                                     };
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(zooKeeperLogPositionManager);
+        if (!zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.start();
+        }
+        executor = Executors.newFixedThreadPool(1);
+        positions = new MapMaker().makeComputingMap(new Function<String, LogPosition>() {
+
+            public LogPosition apply(String destination) {
+                LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
+                if (logPosition == null) {
+                    return nullPosition;
+                } else {
+                    return logPosition;
+                }
+            }
+        });
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.stop();
+        }
+        executor.shutdownNow();
+        positions.clear();
+    }
+
+    public void persistLogPosition(final String destination, final LogPosition logPosition) {
+        super.persistLogPosition(destination, logPosition);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                try {
+                    zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
+                } catch (Exception e) {
+                    logger.error("ERROR # persist to zookeepr has an error", e);
+                }
+            }
+        });
+
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPosition = super.getLatestIndexBy(destination);
+        if (logPosition == nullPosition) {
+            return null;
+        } else {
+            return logPosition;
+        }
+    }
+
+    // ======================== setter / getter ======================
+
+    public void setZooKeeperLogPositionManager(ZooKeeperLogPositionManager zooKeeperLogPositionManager) {
+        this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java

@@ -0,0 +1,111 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于定时刷新的策略的mixed实现
+ * 
+ * @author jianghang 2012-9-12 上午11:18:14
+ * @version 1.0.0
+ */
+public class PeriodMixedLogPositionManager extends MemoryLogPositionManager implements CanalLogPositionManager {
+
+    private static final Logger         logger       = LoggerFactory.getLogger(PeriodMixedLogPositionManager.class);
+    private ZooKeeperLogPositionManager zooKeeperLogPositionManager;
+    private ScheduledExecutorService    executor;
+    @SuppressWarnings("serial")
+    private final LogPosition           nullPosition = new LogPosition() {
+                                                     };
+
+    private long                        period       = 1000;                                                        // 单位ms
+    private Set<String>                 persistTasks;
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(zooKeeperLogPositionManager);
+        if (!zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.start();
+        }
+        executor = Executors.newScheduledThreadPool(1);
+        positions = new MapMaker().makeComputingMap(new Function<String, LogPosition>() {
+
+            public LogPosition apply(String destination) {
+                LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
+                if (logPosition == null) {
+                    return nullPosition;
+                } else {
+                    return logPosition;
+                }
+            }
+        });
+
+        persistTasks = Collections.synchronizedSet(new HashSet<String>());
+
+        // 启动定时工作任务
+        executor.scheduleAtFixedRate(new Runnable() {
+
+            public void run() {
+                List<String> tasks = new ArrayList<String>(persistTasks);
+                for (String destination : tasks) {
+                    try {
+                        // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
+                        zooKeeperLogPositionManager.persistLogPosition(destination, getLatestIndexBy(destination));
+                        persistTasks.remove(destination);
+                    } catch (Throwable e) {
+                        // ignore
+                        logger.error("period update" + destination + " curosr failed!", e);
+                    }
+                }
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.stop();
+        }
+        executor.shutdownNow();
+        positions.clear();
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        persistTasks.add(destination);// 添加到任务队列中进行触发
+        super.persistLogPosition(destination, logPosition);
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPostion = super.getLatestIndexBy(destination);
+        if (logPostion == nullPosition) {
+            return null;
+        } else {
+            return logPostion;
+        }
+    }
+
+    public void setZooKeeperLogPositionManager(ZooKeeperLogPositionManager zooKeeperLogPositionManager) {
+        this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
+    }
+
+    public void setPeriod(long period) {
+        this.period = period;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManager.java

@@ -0,0 +1,57 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 基于zk的实现
+ * 
+ * @author jianghang 2012-7-7 上午10:08:27
+ * @version 1.0.0
+ */
+public class ZooKeeperLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    private ZkClientx zkClientx;
+
+    public void start() {
+        super.start();
+        Assert.notNull(zkClientx);
+    }
+
+    public void stop() {
+        super.stop();
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        String path = ZookeeperPathUtils.getParsePath(destination);
+        byte[] data = zkClientx.readData(path, true);
+        if (data == null || data.length == 0) {
+            return null;
+        }
+
+        return JsonUtils.unmarshalFromByte(data, LogPosition.class);
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        String path = ZookeeperPathUtils.getParsePath(destination);
+        byte[] data = JsonUtils.marshalToByte(logPosition);
+        try {
+            zkClientx.writeData(path, data);
+        } catch (ZkNoNodeException e) {
+            zkClientx.createPersistent(path, data, true);
+        }
+    }
+
+    // ================== setter / getter =================
+
+    public void setZkClientx(ZkClientx zkClientx) {
+        this.zkClientx = zkClientx;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java

@@ -0,0 +1,127 @@
+package com.alibaba.otter.canal.parse.support;
+
+import java.net.InetSocketAddress;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+
+/**
+ * 数据库认证信息
+ * 
+ * @author jianghang 2012-7-11 上午11:22:19
+ * @version 1.0.0
+ */
+public class AuthenticationInfo {
+
+    private InetSocketAddress address;            // 主库信息
+    private String            username;           // 帐号
+    private String            password;           // 密码
+    private String            defaultDatabaseName; // 默认链接的数据库
+
+    public AuthenticationInfo(){
+        super();
+    }
+
+    public AuthenticationInfo(InetSocketAddress address, String username, String password){
+        this(address, username, password, "");
+    }
+
+    public AuthenticationInfo(InetSocketAddress address, String username, String password, String defaultDatabaseName){
+        this.address = address;
+        this.username = username;
+        this.password = password;
+        this.defaultDatabaseName = defaultDatabaseName;
+    }
+
+    public InetSocketAddress getAddress() {
+        return address;
+    }
+
+    public void setAddress(InetSocketAddress address) {
+        this.address = address;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getDefaultDatabaseName() {
+        return defaultDatabaseName;
+    }
+
+    public void setDefaultDatabaseName(String defaultDatabaseName) {
+        this.defaultDatabaseName = defaultDatabaseName;
+    }
+
+    @Override
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, ToStringStyle.DEFAULT_STYLE);
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((address == null) ? 0 : address.hashCode());
+        result = prime * result + ((defaultDatabaseName == null) ? 0 : defaultDatabaseName.hashCode());
+        result = prime * result + ((password == null) ? 0 : password.hashCode());
+        result = prime * result + ((username == null) ? 0 : username.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof AuthenticationInfo)) {
+            return false;
+        }
+        AuthenticationInfo other = (AuthenticationInfo) obj;
+        if (address == null) {
+            if (other.address != null) {
+                return false;
+            }
+        } else if (!address.equals(other.address)) {
+            return false;
+        }
+        if (defaultDatabaseName == null) {
+            if (other.defaultDatabaseName != null) {
+                return false;
+            }
+        } else if (!defaultDatabaseName.equals(other.defaultDatabaseName)) {
+            return false;
+        }
+        if (password == null) {
+            if (other.password != null) {
+                return false;
+            }
+        } else if (!password.equals(other.password)) {
+            return false;
+        }
+        if (username == null) {
+            if (other.username != null) {
+                return false;
+            }
+        } else if (!username.equals(other.username)) {
+            return false;
+        }
+        return true;
+    }
+
+}

parse/src/main/java/com/alibaba/otter/canal/parse/support/HaAuthenticationInfo.java

@@ -0,0 +1,36 @@
+package com.alibaba.otter.canal.parse.support;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * @author zebin.xuzb 2012-9-26 下午3:08:11
+ * @version 1.0.0
+ */
+public class HaAuthenticationInfo {
+
+    private AuthenticationInfo       master;
+    private List<AuthenticationInfo> slavers = new ArrayList<AuthenticationInfo>();
+
+    public AuthenticationInfo getMaster() {
+        return master;
+    }
+
+    public void setMaster(AuthenticationInfo master) {
+        this.master = master;
+    }
+
+    public List<AuthenticationInfo> getSlavers() {
+        return slavers;
+    }
+
+    public void addSlaver(AuthenticationInfo slaver) {
+        this.slavers.add(slaver);
+    }
+
+    public void addSlavers(Collection<AuthenticationInfo> slavers) {
+        this.slavers.addAll(slavers);
+    }
+
+}

parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java

@@ -0,0 +1,101 @@
+package com.alibaba.otter.canal.parse;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class DirectLogFetcherTest {
+
+    @Test
+    public void testSimple() {
+        DirectLogFetcher fetcher = new DirectLogFetcher();
+        try {
+            MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
+            connector.connect();
+            sendBinlogDump(connector, "mysql-bin.001016", 4L, 3);
+
+            fetcher.start(connector.getChannel());
+
+            LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+            LogContext context = new LogContext();
+            while (fetcher.fetch()) {
+                LogEvent event = null;
+                event = decoder.decode(fetcher, context);
+
+                if (event == null) {
+                    throw new RuntimeException("parse failed");
+                }
+
+                int eventType = event.getHeader().getType();
+                switch (eventType) {
+                    case LogEvent.ROTATE_EVENT:
+                        // binlogFileName = ((RotateLogEvent)
+                        // event).getFilename();
+                        break;
+                    case LogEvent.WRITE_ROWS_EVENT_V1:
+                    case LogEvent.WRITE_ROWS_EVENT:
+                        // parseRowsEvent((WriteRowsLogEvent) event);
+                        break;
+                    case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.UPDATE_ROWS_EVENT:
+                        // parseRowsEvent((UpdateRowsLogEvent) event);
+                        break;
+                    case LogEvent.DELETE_ROWS_EVENT_V1:
+                    case LogEvent.DELETE_ROWS_EVENT:
+                        // parseRowsEvent((DeleteRowsLogEvent) event);
+                        break;
+                    case LogEvent.QUERY_EVENT:
+                        // parseQueryEvent((QueryLogEvent) event);
+                        break;
+                    case LogEvent.ROWS_QUERY_LOG_EVENT:
+                        // parseRowsQueryEvent((RowsQueryLogEvent) event);
+                        break;
+                    case LogEvent.ANNOTATE_ROWS_EVENT:
+                        break;
+                    case LogEvent.XID_EVENT:
+                        break;
+                    default:
+                        break;
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                fetcher.close();
+            } catch (IOException e) {
+                Assert.fail(e.getMessage());
+            }
+        }
+
+    }
+
+    private void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
+                                                                                                                  throws IOException {
+        BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
+        binlogDumpCmd.binlogFileName = binlogfilename;
+        binlogDumpCmd.binlogPosition = binlogPosition;
+        binlogDumpCmd.slaveServerId = slaveId;
+        byte[] cmdBody = binlogDumpCmd.toBytes();
+
+        HeaderPacket binlogDumpHeader = new HeaderPacket();
+        binlogDumpHeader.setPacketBodyLength(cmdBody.length);
+        binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
+        PacketManager.write(connector.getChannel(), new ByteBuffer[] { ByteBuffer.wrap(binlogDumpHeader.toBytes()),
+                ByteBuffer.wrap(cmdBody) });
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java

@@ -0,0 +1,59 @@
+package com.alibaba.otter.canal.parse.helper;
+
+/**
+ * 用于检查超时, 主要用于启动服务以后，如果在指定的时间内没有响应，则自动退出
+ * 
+ * @author: yuanzu Date: 12-9-26 Time: 上午10:55
+ */
+public class TimeoutChecker {
+
+    /**
+     * 最后一次动作的时间
+     */
+    private long              lastTouch;
+    /**
+     * 超时时间
+     */
+    private long              timeoutMillis;
+
+    /**
+     * 运行标志
+     */
+    private boolean           running                = true;
+
+    /**
+     * default 3s
+     */
+    private static final long DEFAULT_TIMEOUT_MILLIS = 3 * 1000;
+
+    public TimeoutChecker(long timeoutMillis){
+        this.timeoutMillis = timeoutMillis;
+        touch();
+    }
+
+    public TimeoutChecker(){
+        this(DEFAULT_TIMEOUT_MILLIS);
+    }
+
+    /**
+     * 更新
+     */
+    public void touch() {
+        this.lastTouch = System.currentTimeMillis();
+    }
+
+    /**
+     * 等待空闲
+     * 
+     * @throws InterruptedException
+     */
+    public void waitForIdle() throws InterruptedException {
+        while (this.running && (System.currentTimeMillis() - this.lastTouch) < this.timeoutMillis) {
+            Thread.sleep(50);
+        }
+    }
+
+    public void stop() {
+        this.running = false;
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java

@@ -0,0 +1,138 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+
+public class EventTransactionBufferTest {
+
+    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+    private static final String messgae     = "{0} [{1}:{2}:{3}] {4}.{5}";
+
+    @Test
+    public void testTransactionFlush() {
+        final int bufferSize = 64;
+        final int transactionSize = 5;
+        EventTransactionBuffer buffer = new EventTransactionBuffer();
+        buffer.setBufferSize(bufferSize);
+        buffer.setFlushCallback(new TransactionFlushCallback() {
+
+            public void flush(List<Entry> transaction) throws InterruptedException {
+                Assert.assertEquals(transactionSize, transaction.size());
+                System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+                for (Entry data : transaction) {
+
+                    CanalEntry.Header header = data.getHeader();
+                    Date date = new Date(header.getExecuteTime());
+                    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+                    if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || data.getEntryType() == EntryType.TRANSACTIONEND) {
+                        System.out.println(data.getEntryType());
+
+                    } else {
+                        System.out.println(MessageFormat.format(messgae, new Object[] {
+                                Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(),
+                                format.format(date), header.getSchemaName(), header.getTableName() }));
+                    }
+
+                }
+                System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
+            }
+        });
+        buffer.start();
+
+        try {
+            for (int i = 0; i < transactionSize * 10; i++) {
+                if (i % transactionSize == 0) {
+                    buffer.add(buildEntry("1", 1L + i, 40L + i, EntryType.TRANSACTIONBEGIN));
+                } else if ((i + 1) % transactionSize == 0) {
+                    buffer.add(buildEntry("1", 1L + i, 40L + i, EntryType.TRANSACTIONEND));
+                } else {
+                    buffer.add(buildEntry("1", 1L + i, 40L + i));
+                }
+            }
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        buffer.stop();
+    }
+
+    @Test
+    public void testForceFlush() {
+        final int bufferSize = 64;
+        EventTransactionBuffer buffer = new EventTransactionBuffer();
+        buffer.setBufferSize(bufferSize);
+        buffer.setFlushCallback(new TransactionFlushCallback() {
+
+            public void flush(List<Entry> transaction) throws InterruptedException {
+                Assert.assertEquals(bufferSize, transaction.size());
+                System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+                for (Entry data : transaction) {
+
+                    CanalEntry.Header header = data.getHeader();
+                    Date date = new Date(header.getExecuteTime());
+                    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+                    if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || data.getEntryType() == EntryType.TRANSACTIONEND) {
+                        // System.out.println(MessageFormat.format(messgae, new Object[] {
+                        // Thread.currentThread().getName(),
+                        // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                        // data.getEntry().getEntryType(), "" }));
+                        System.out.println(data.getEntryType());
+
+                    } else {
+                        System.out.println(MessageFormat.format(messgae, new Object[] {
+                                Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(),
+                                format.format(date), header.getSchemaName(), header.getTableName() }));
+                    }
+
+                }
+                System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
+            }
+        });
+        buffer.start();
+
+        try {
+            for (int i = 0; i < bufferSize * 2 + 1; i++) {
+                buffer.add(buildEntry("1", 1L + i, 40L + i));
+            }
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        buffer.stop();
+    }
+
+    private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        return entryBuilder.build();
+    }
+
+    private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        entryBuilder.setEntryType(type);
+        return entryBuilder.build();
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java

@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
+
+public class TableMetaCacheTest {
+
+    @Test
+    public void testSimple() {
+
+        MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
+        try {
+            connection.connect();
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        TableMetaCache cache = new TableMetaCache(connection);
+        TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
+        Assert.assertNotNull(meta);
+        for (FieldMeta field : meta.getFileds()) {
+            System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : "
+                               + field.isNullable());
+        }
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java

@@ -0,0 +1,158 @@
+package com.alibaba.otter.canal.parse.inbound.group;
+
+import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+public class DummyEventStore implements CanalEventStore<Event> {
+
+    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+    private static final String messgae     = "{0} [{1}:{2}:{3}] {4} {5}.{6}";
+
+    public void ack(Position position) throws CanalStoreException {
+
+    }
+
+    public Events get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
+        return null;
+    }
+
+    public Events get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
+                                                                                 CanalStoreException {
+        return null;
+    }
+
+    public Position getFirstPosition() throws CanalStoreException {
+        return null;
+    }
+
+    public Position getLatestPosition() throws CanalStoreException {
+        return null;
+    }
+
+    public void rollback() throws CanalStoreException {
+
+    }
+
+    public Events tryGet(Position start, int batchSize) throws CanalStoreException {
+        return null;
+    }
+
+    public boolean isStart() {
+        return false;
+    }
+
+    public void start() {
+
+    }
+
+    public void stop() {
+
+    }
+
+    public void cleanAll() throws CanalStoreException {
+    }
+
+    public void cleanUntil(Position position) throws CanalStoreException {
+
+    }
+
+    public void put(Event data) throws InterruptedException, CanalStoreException {
+        put(Arrays.asList(data));
+    }
+
+    public boolean put(Event data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        return put(Arrays.asList(data), timeout, unit);
+    }
+
+    public boolean tryPut(Event data) throws CanalStoreException {
+        return tryPut(Arrays.asList(data));
+    }
+
+    public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
+        for (Event data : datas) {
+            CanalEntry.Header header = data.getEntry().getHeader();
+            Date date = new Date(header.getExecuteTime());
+            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+            if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
+                || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
+                // System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
+                // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                // data.getEntry().getEntryType(), "" }));
+                System.out.println(data.getEntry().getEntryType());
+
+            } else {
+                System.out.println(MessageFormat.format(messgae,
+                                                        new Object[] { Thread.currentThread().getName(),
+                                                                header.getLogfileName(),
+                                                                String.valueOf(header.getLogfileOffset()),
+                                                                format.format(date), header.getEventType(),
+                                                                header.getSchemaName(), header.getTableName() }));
+            }
+        }
+    }
+
+    public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        for (Event data : datas) {
+            CanalEntry.Header header = data.getEntry().getHeader();
+            Date date = new Date(header.getExecuteTime());
+            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+            if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
+                || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
+                // System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
+                // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                // data.getEntry().getEntryType(), "" }));
+                System.out.println(data.getEntry().getEntryType());
+
+            } else {
+                System.out.println(MessageFormat.format(messgae,
+                                                        new Object[] { Thread.currentThread().getName(),
+                                                                header.getLogfileName(),
+                                                                String.valueOf(header.getLogfileOffset()),
+                                                                format.format(date), header.getEventType(),
+                                                                header.getSchemaName(), header.getTableName() }));
+            }
+        }
+        return true;
+    }
+
+    public boolean tryPut(List<Event> datas) throws CanalStoreException {
+        System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+        for (Event data : datas) {
+
+            CanalEntry.Header header = data.getEntry().getHeader();
+            Date date = new Date(header.getExecuteTime());
+            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+            if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
+                || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
+                // System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
+                // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                // data.getEntry().getEntryType(), "" }));
+                System.out.println(data.getEntry().getEntryType());
+
+            } else {
+                System.out.println(MessageFormat.format(messgae,
+                                                        new Object[] { Thread.currentThread().getName(),
+                                                                header.getLogfileName(),
+                                                                String.valueOf(header.getLogfileOffset()),
+                                                                format.format(date), header.getEventType(),
+                                                                header.getSchemaName(), header.getTableName() }));
+            }
+
+        }
+        System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
+        return true;
+    }
+
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java

@@ -0,0 +1,103 @@
+package com.alibaba.otter.canal.parse.inbound.group;
+
+import java.net.InetSocketAddress;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.AbstractBinlogParser;
+import com.alibaba.otter.canal.parse.inbound.BinlogParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.entry.EntryEventSink;
+import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class GroupEventPaserTest {
+
+    private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private static final String USERNAME      = "xxxxx";
+    private static final String PASSWORD      = "xxxxx";
+
+    @Test
+    public void testMysqlWithMysql() {
+        // MemoryEventStoreWithBuffer eventStore = new
+        // MemoryEventStoreWithBuffer();
+        // eventStore.setBufferSize(8196);
+
+        GroupEventSink eventSink = new GroupEventSink(3);
+        eventSink.setFilterTransactionEntry(false);
+        eventSink.setEventStore(new DummyEventStore());
+        eventSink.start();
+
+        // 构造第一个mysql
+        MysqlEventParser mysqlEventPaser1 = buildEventParser(3344);
+        mysqlEventPaser1.setEventSink(eventSink);
+        // 构造第二个mysql
+        MysqlEventParser mysqlEventPaser2 = buildEventParser(3345);
+        mysqlEventPaser2.setEventSink(eventSink);
+        // 构造第二个mysql
+        MysqlEventParser mysqlEventPaser3 = buildEventParser(3346);
+        mysqlEventPaser3.setEventSink(eventSink);
+        // 启动
+        mysqlEventPaser1.start();
+        mysqlEventPaser2.start();
+        mysqlEventPaser3.start();
+
+        try {
+            Thread.sleep(30 * 10 * 1000L);
+        } catch (InterruptedException e) {
+        }
+
+        mysqlEventPaser1.stop();
+        mysqlEventPaser2.stop();
+        mysqlEventPaser3.stop();
+    }
+
+    private MysqlEventParser buildEventParser(int slaveId) {
+        MysqlEventParser mysqlEventPaser = new MysqlEventParser();
+        EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+        mysqlEventPaser.setDestination("group-" + slaveId);
+        mysqlEventPaser.setSlaveId(slaveId);
+        mysqlEventPaser.setDetectingEnable(false);
+        mysqlEventPaser.setDetectingSQL(DETECTING_SQL);
+        mysqlEventPaser.setMasterInfo(buildAuthentication());
+        mysqlEventPaser.setMasterPosition(defaultPosition);
+        mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
+        mysqlEventPaser.setEventSink(new EntryEventSink());
+        mysqlEventPaser.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                // System.out.println(logPosition);
+            }
+
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+        return mysqlEventPaser;
+    }
+
+    private BinlogParser buildParser(AuthenticationInfo info) {
+        return new AbstractBinlogParser<LogEvent>() {
+
+            public Entry parse(LogEvent event) throws CanalParseException {
+                // return _parser.parse(event);
+                return null;
+            }
+        };
+    }
+
+    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+        return new EntryPosition(binlogFile, offest, timestamp);
+    }
+
+    private AuthenticationInfo buildAuthentication() {
+        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java

@@ -0,0 +1,111 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class LocalBinlogDumpTest {
+
+    @Test
+    public void testSimple() {
+        String directory = "/home/jianghang/tmp/binlog";
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        final EntryPosition startPosition = new EntryPosition("mysql-bin.000006", 4L);
+
+        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
+        controller.setConnectionCharset(Charset.forName("UTF-8"));
+        controller.setDirectory(directory);
+        controller.setMasterPosition(startPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException,
+                                                                                                        InterruptedException {
+
+                for (Entry entry : entrys) {
+                    if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || entry.getEntryType() == EntryType.TRANSACTIONEND) {
+                        continue;
+                    }
+
+                    if (entry.getEntryType() == EntryType.ROWDATA) {
+                        RowChange rowChage = null;
+                        try {
+                            rowChage = RowChange.parseFrom(entry.getStoreValue());
+                        } catch (Exception e) {
+                            throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+                                                       + entry.toString(), e);
+                        }
+
+                        EventType eventType = rowChage.getEventType();
+                        System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
+                            entry.getHeader().getLogfileName(),
+                            entry.getHeader().getLogfileOffset(),
+                            entry.getHeader().getSchemaName(),
+                            entry.getHeader().getTableName(),
+                            eventType));
+
+                        for (RowData rowData : rowChage.getRowDatasList()) {
+                            if (eventType == EventType.DELETE) {
+                                print(rowData.getBeforeColumnsList());
+                            } else if (eventType == EventType.INSERT) {
+                                print(rowData.getAfterColumnsList());
+                            } else {
+                                System.out.println("-------> before");
+                                print(rowData.getBeforeColumnsList());
+                                System.out.println("-------> after");
+                                print(rowData.getAfterColumnsList());
+                            }
+                        }
+                    }
+                }
+
+                return true;
+            }
+
+        });
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        try {
+            Thread.sleep(100 * 1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        controller.stop();
+    }
+
+    private void print(List<Column> columns) {
+        for (Column column : columns) {
+            System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
+        }
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java

@@ -0,0 +1,238 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class LocalBinlogEventParserTest {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private static final String USERNAME      = "xxxxx";
+    private static final String PASSWORD      = "xxxxx";
+    private String              directory;
+
+    @Before
+    public void setUp() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
+        File dummyFile = new File(url.getFile());
+        directory = new File(dummyFile.getParent() + "/binlog").getPath();
+    }
+
+    @Test
+    public void test_position() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker();
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setDirectory(directory);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                entryCount.incrementAndGet();
+
+                for (Entry entry : entrys) {
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        assertEquals(entryPosition, defaultPosition);
+    }
+
+    @Test
+    public void test_timestamp() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(300 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, 1322803601000L);
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setDirectory(directory);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            @Override
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+        assertTrue(entryPosition.getPosition() <= 6163L);
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    @Test
+    public void test_no_position() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000);
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000002",
+            null,
+            new Date().getTime() + 1000 * 1000L);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setDirectory(directory);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        // assertEquals(entryPosition.getJournalName(), "mysql-bin.000002");
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+        return new EntryPosition(binlogFile, offest, timestamp);
+    }
+
+    private AuthenticationInfo buildAuthentication() {
+        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java

@@ -0,0 +1,114 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class MysqlDumpTest {
+
+    @Test
+    public void testSimple() {
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L);
+
+        controller.setConnectionCharset(Charset.forName("UTF-8"));
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(false);
+        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
+        controller.setMasterPosition(startPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException,
+                                                                                                        InterruptedException {
+
+                for (Entry entry : entrys) {
+                    if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || entry.getEntryType() == EntryType.TRANSACTIONEND
+                        || entry.getEntryType() == EntryType.HEARTBEAT) {
+                        continue;
+                    }
+
+                    RowChange rowChage = null;
+                    try {
+                        rowChage = RowChange.parseFrom(entry.getStoreValue());
+                    } catch (Exception e) {
+                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+                                                   + entry.toString(), e);
+                    }
+
+                    EventType eventType = rowChage.getEventType();
+                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
+                        entry.getHeader().getLogfileName(),
+                        entry.getHeader().getLogfileOffset(),
+                        entry.getHeader().getSchemaName(),
+                        entry.getHeader().getTableName(),
+                        eventType));
+
+                    if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
+                        System.out.println(" sql ----> " + rowChage.getSql());
+                    }
+
+                    for (RowData rowData : rowChage.getRowDatasList()) {
+                        if (eventType == EventType.DELETE) {
+                            print(rowData.getBeforeColumnsList());
+                        } else if (eventType == EventType.INSERT) {
+                            print(rowData.getAfterColumnsList());
+                        } else {
+                            System.out.println("-------> before");
+                            print(rowData.getBeforeColumnsList());
+                            System.out.println("-------> after");
+                            print(rowData.getAfterColumnsList());
+                        }
+                    }
+                }
+
+                return true;
+            }
+
+        });
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        try {
+            Thread.sleep(100 * 1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        controller.stop();
+    }
+
+    private void print(List<Column> columns) {
+        for (Column column : columns) {
+            System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
+        }
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java

@@ -0,0 +1,308 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class MysqlEventParserTest {
+
+    private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private static final String USERNAME      = "xxxxx";
+    private static final String PASSWORD      = "xxxxx";
+
+    @Test
+    public void test_position() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker();
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(true);
+        controller.setDetectingSQL(DETECTING_SQL);
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            @Override
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        Assert.assertEquals(entryPosition, defaultPosition);
+    }
+
+    @Test
+    public void test_timestamp() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition(null, null, 1322803601000L);
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(true);
+        controller.setDetectingSQL(DETECTING_SQL);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setMasterPosition(defaultPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            @Override
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+        assertTrue(entryPosition.getPosition() <= 6163L);
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    @Test
+    public void test_ha() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(false);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setMasterPosition(defaultPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            @Override
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            public LogPosition getLatestIndexBy(String destination) {
+                LogPosition masterLogPosition = new LogPosition();
+                masterLogPosition.setIdentity(new LogIdentity(new InetSocketAddress("127.0.0.1", 3306), 1234L));
+                masterLogPosition.setPostion(new EntryPosition(1322803601000L));
+                return masterLogPosition;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+        assertTrue(entryPosition.getPosition() <= 6163L);
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    @Test
+    public void test_no_position() throws InterruptedException { // 在某个文件下，找不到对应的timestamp数据，会使用106L
+                                                                 // position进行数据抓取
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 60 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001",
+            null,
+            new Date().getTime() + 1000 * 1000L);
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(false);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setMasterPosition(defaultPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
+
+            @Override
+            public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    // String logfilename = entry.getHeader().getLogfileName();
+                    // long logfileoffset =
+                    // entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    // entryPosition.setJournalName(logfilename);
+                    // entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        // Assert.assertEquals(logfilename, "mysql-bin.000001");
+        // Assert.assertEquals(106L, logfileoffset);
+        assertTrue(entryPosition.getTimestamp() < defaultPosition.getTimestamp());
+    }
+
+    // ======================== helper method =======================
+
+    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+        return new EntryPosition(binlogFile, offest, timestamp);
+    }
+
+    private AuthenticationInfo buildAuthentication() {
+        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java

@@ -0,0 +1,170 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult;
+
+public class SimpleDdlParserTest {
+
+    @Test
+    public void testCreate() {
+        String queryString = "CREATE TABLE retl_mark ( `ID` int(11)";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE IF NOT EXISTS retl.retl_mark ( `ID` int(11)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE IF NOT EXISTS `retl_mark` ( `ID` int(11)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE IF NOT EXISTS `retl.retl_mark` ( `ID` int(11)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE  `retl`.`retl_mark` (\n  `ID` int(10) unsigned NOT NULL";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE  `retl`.`retl_mark`(\n  `ID` int(10) unsigned NOT NULL";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE table `bak591`.`j_order_log_back_201309` like j_order_log";
+        result = SimpleDdlParser.parse(queryString, "bak");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("bak591", result.getSchemaName());
+        Assert.assertEquals("j_order_log_back_201309", result.getTableName());
+
+        queryString = "CREATE TABLE `bak591`.`cm_settle_incash` (    `batch_id` bigint(20) NOT NULL  DEFAULT '0.00'";
+        result = SimpleDdlParser.parse(queryString, "bak");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("bak591", result.getSchemaName());
+        Assert.assertEquals("cm_settle_incash", result.getTableName());
+    }
+
+    @Test
+    public void testDrop() {
+        String queryString = "DROP TABLE retl_mark";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "DROP TABLE IF EXISTS retl.retl_mark;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "DROP TABLE IF EXISTS \n `retl.retl_mark` /;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "DROP /*!40005 TEMPORARY */  /*!40005 TEMPORARY */ TABLE IF EXISTS `temp_bond_keys`.`temp_bond_key_id`;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("temp_bond_keys", result.getSchemaName());
+        Assert.assertEquals("temp_bond_key_id", result.getTableName());
+    }
+
+    @Test
+    public void testAlert() {
+        String queryString = "alter table retl_mark drop index emp_name";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "alter table retl.retl_mark drop index emp_name";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "alter table \n `retl.retl_mark` drop index emp_name;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+    }
+
+    @Test
+    public void testTruncate() {
+        String queryString = "truncate table retl_mark";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "truncate table retl.retl_mark";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "truncate \n  `retl.retl_mark` ";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+    }
+
+    @Test
+    public void testRename() {
+        String queryString = "rename table retl_mark to retl_mark2";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getOriSchemaName());
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getOriTableName());
+        Assert.assertEquals("retl_mark2", result.getTableName());
+
+        queryString = "rename table retl.retl_mark to retl2.retl_mark2";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getOriSchemaName());
+        Assert.assertEquals("retl2", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getOriTableName());
+        Assert.assertEquals("retl_mark2", result.getTableName());
+
+        queryString = "rename \n table \n `retl`.`retl_mark` to `retl2.retl_mark2`;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getOriSchemaName());
+        Assert.assertEquals("retl2", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getOriTableName());
+        Assert.assertEquals("retl_mark2", result.getTableName());
+    }
+
+    @Test
+    public void testIndex() {
+        String queryString = "CREATE UNIQUE INDEX index_1 ON retl_mark(id,x)";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "create index idx_qca_cid_mcid on q_contract_account (contract_id,main_contract_id)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("q_contract_account", result.getTableName());
+
+        queryString = "DROP INDEX index_str ON retl_mark";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManagerTest.java

@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+
+import junit.framework.Assert;
+
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class AbstractLogPositionManagerTest extends AbstractZkTest {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+
+    public LogPosition doTest(CanalLogPositionManager logPositionManager) {
+        LogPosition getPosition = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertNull(getPosition);
+
+        LogPosition postion1 = buildPosition(1);
+        logPositionManager.persistLogPosition(destination, postion1);
+        LogPosition getPosition1 = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertEquals(postion1, getPosition1);
+
+        LogPosition postion2 = buildPosition(2);
+        logPositionManager.persistLogPosition(destination, postion2);
+        LogPosition getPosition2 = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertEquals(postion2, getPosition2);
+        return postion2;
+    }
+
+    protected LogPosition buildPosition(int number) {
+        LogPosition position = new LogPosition();
+        position.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        position.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
+        return position;
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java

@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.junit.Assert;
+
+public class AbstractZkTest {
+
+    protected String destination = "ljhtest1";
+    protected String cluster1    = "127.0.0.1:2188";
+    protected String cluster2    = "127.0.0.1:2188,127.0.0.1:2188";
+
+    public void sleep(long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java

@@ -0,0 +1,49 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.io.File;
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class FileMixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private static final String tmp     = System.getProperty("java.io.tmpdir", "/tmp");
+    private static final File   dataDir = new File(tmp, "canal");
+
+    @Before
+    public void setUp() {
+        try {
+            FileUtils.deleteDirectory(dataDir);
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+
+    @Test
+    public void testAll() {
+        FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager();
+        logPositionManager.setDataDir(dataDir);
+        logPositionManager.setPeriod(100);
+        logPositionManager.start();
+
+        LogPosition position2 = doTest(logPositionManager);
+        sleep(1500);
+
+        FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager();
+        logPositionManager2.setDataDir(dataDir);
+        logPositionManager2.setPeriod(100);
+        logPositionManager2.start();
+
+        LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
+        Assert.assertEquals(position2, getPosition2);
+
+        logPositionManager.stop();
+        logPositionManager2.stop();
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java

@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.junit.Test;
+
+public class MemoryLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    @Test
+    public void testAll() {
+        MemoryLogPositionManager logPositionManager = new MemoryLogPositionManager();
+        logPositionManager.start();
+        doTest(logPositionManager);
+        logPositionManager.stop();
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java

@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.meta.MixedMetaManager;
+import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class MetaLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private ZkClientx           zkclientx     = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        MixedMetaManager metaManager = new MixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+
+        MetaLogPositionManager logPositionManager = new MetaLogPositionManager();
+        logPositionManager.setMetaManager(metaManager);
+        logPositionManager.start();
+        // 构建meta信息
+        ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
+        metaManager.subscribe(client1);
+
+        PositionRange range1 = buildRange(1);
+        metaManager.updateCursor(client1, range1.getEnd());
+
+        PositionRange range2 = buildRange(2);
+        metaManager.updateCursor(client1, range2.getEnd());
+
+        ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
+        metaManager.subscribe(client2);
+
+        PositionRange range3 = buildRange(3);
+        metaManager.updateCursor(client2, range3.getEnd());
+
+        PositionRange range4 = buildRange(4);
+        metaManager.updateCursor(client2, range4.getEnd());
+
+        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertEquals(range2.getEnd(), logPosition);
+
+        metaManager.stop();
+        logPositionManager.stop();
+    }
+
+    private PositionRange<LogPosition> buildRange(int number) {
+        LogPosition start = new LogPosition();
+        start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
+
+        LogPosition end = new LogPosition();
+        end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
+        return new PositionRange<LogPosition>(start, end);
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.parse.index;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class MixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        MixedLogPositionManager logPositionManager = new MixedLogPositionManager();
+
+        ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
+        zookeeperLogPositionManager.setZkClientx(zkclientx);
+
+        logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager.start();
+
+        LogPosition position2 = doTest(logPositionManager);
+        sleep(1000);
+
+        MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager();
+        logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager2.start();
+
+        LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
+        Assert.assertEquals(position2, getPosition2);
+
+        logPositionManager.stop();
+        logPositionManager2.stop();
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.parse.index;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class PeriodMixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        PeriodMixedLogPositionManager logPositionManager = new PeriodMixedLogPositionManager();
+
+        ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
+        zookeeperLogPositionManager.setZkClientx(zkclientx);
+
+        logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager.start();
+
+        LogPosition position2 = doTest(logPositionManager);
+        sleep(1500);
+
+        PeriodMixedLogPositionManager logPositionManager2 = new PeriodMixedLogPositionManager();
+        logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager2.start();
+
+        LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
+        Assert.assertEquals(position2, getPosition2);
+
+        logPositionManager.stop();
+        logPositionManager2.stop();
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+
+public class ZooKeeperLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        ZooKeeperLogPositionManager logPositionManager = new ZooKeeperLogPositionManager();
+        logPositionManager.setZkClientx(zkclientx);
+        logPositionManager.start();
+
+        doTest(logPositionManager);
+        logPositionManager.stop();
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java

@@ -0,0 +1,11 @@
+package com.alibaba.otter.canal.parse.stub;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+
+public abstract class AbstractCanalEventSinkTest<T> extends AbstractCanalLifeCycle implements CanalEventSink<T> {
+
+    public void interrupt() {
+        // do nothing
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java

@@ -0,0 +1,8 @@
+package com.alibaba.otter.canal.parse.stub;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+
+public abstract class AbstractCanalLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+}

parse/src/test/resources/dummy.txt

@@ -0,0 +1 @@
+本文件仅仅为定位绝对路径使用