chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pyf2

Browse Source
This commit is contained in:
Payon
2018-08-07 19:29:27 +08:00
committed by GitHub
parent 02868d8fec 23f7b3b5d8
commit b1dc5bd59f
168 changed files with 19475 additions and 11726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
parse/pom.xml
View File

@@ -73,6 +73,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
<!-- test dependency -->
<dependency>
<groupId>junit</groupId>

85
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
View File

@@ -88,12 +88,19 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
protected TimerTask heartBeatTimerTask;
protected Throwable exception = null;
protected boolean isGTIDMode = false; // 是否是GTID模式
protected boolean isGTIDMode = false; // 是否是GTID模式
protected boolean parallel = true; // 是否开启并行解析模式
protected Integer parallelThreadSize = Runtime.getRuntime()
.availableProcessors() * 60 / 100; // 60%的能力跑解析,剩余部分处理网络
protected int parallelBufferSize = 256; // 必须为2的幂
protected MultiStageCoprocessor multiStageCoprocessor;
protected abstract BinlogParser buildParser();
protected abstract ErosaConnection buildErosaConnection();
protected abstract MultiStageCoprocessor buildMultiStageCoprocessor();
protected abstract EntryPosition findStartPosition(ErosaConnection connection) throws IOException;
protected void preDump(ErosaConnection connection) {
@@ -217,19 +224,39 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
};
// 4. 开始dump数据
// 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
if (isGTIDMode()) {
erosaConnection.dump(MysqlGTIDSet.parse(startPosition.getGtid()), sinkHandler);
} else {
if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
if (parallel) {
// build stage processor
multiStageCoprocessor = buildMultiStageCoprocessor();
multiStageCoprocessor.start();
if (isGTIDMode()) {
// 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
erosaConnection.dump(MysqlGTIDSet.parse(startPosition.getGtid()), multiStageCoprocessor);
} else {
erosaConnection.dump(startPosition.getJournalName(),
if (StringUtils.isEmpty(startPosition.getJournalName())
&& startPosition.getTimestamp() != null) {
erosaConnection.dump(startPosition.getTimestamp(), multiStageCoprocessor);
} else {
erosaConnection.dump(startPosition.getJournalName(),
startPosition.getPosition(),
multiStageCoprocessor);
}
}
} else {
if (isGTIDMode()) {
// 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
erosaConnection.dump(MysqlGTIDSet.parse(startPosition.getGtid()), sinkHandler);
} else {
if (StringUtils.isEmpty(startPosition.getJournalName())
&& startPosition.getTimestamp() != null) {
erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
} else {
erosaConnection.dump(startPosition.getJournalName(),
startPosition.getPosition(),
sinkHandler);
}
}
}
} catch (TableIdNotFoundException e) {
exception = e;
// 特殊处理TableIdNotFound异常,出现这样的异常，一种可能就是起始的position是一个事务当中，导致tablemap
@@ -275,6 +302,14 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
eventSink.interrupt();
transactionBuffer.reset();// 重置一下缓冲队列，重新记录数据
binlogParser.reset();// 重新置位
if (multiStageCoprocessor != null) {
// 处理 RejectedExecutionException
try {
multiStageCoprocessor.reset();
} catch (Throwable t) {
logger.debug("multi processor rejected:", t);
}
}
if (running) {
// sleep一段时间再进行重试
@@ -301,6 +336,11 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
stopHeartBeat(); // 先停止心跳
parseThread.interrupt(); // 尝试中断
eventSink.interrupt();
if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
multiStageCoprocessor.stop();
}
try {
parseThread.join();// 等待其结束
} catch (InterruptedException e) {
@@ -548,4 +588,31 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
public void setIsGTIDMode(boolean isGTIDMode) {
this.isGTIDMode = isGTIDMode;
}
public boolean isParallel() {
return parallel;
}
public void setParallel(boolean parallel) {
this.parallel = parallel;
}
public int getParallelThreadSize() {
return parallelThreadSize;
}
public void setParallelThreadSize(Integer parallelThreadSize) {
if (parallelThreadSize != null) {
this.parallelThreadSize = parallelThreadSize;
}
}
public Integer getParallelBufferSize() {
return parallelBufferSize;
}
public void setParallelBufferSize(int parallelBufferSize) {
this.parallelBufferSize = parallelBufferSize;
}
}

16
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
View File

@@ -1,9 +1,9 @@
package com.alibaba.otter.canal.parse.inbound;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
/**
* 通用的Erosa的链接接口, 用于一般化处理mysql/oracle的解析过程
*
@@ -28,12 +28,16 @@ public interface ErosaConnection {
/**
* 通过GTID同步binlog
*
* @param gtidSet
* @param func
* @throws IOException
*/
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException;
// -------------
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException;
public void dump(long timestamp, MultiStageCoprocessor coprocessor) throws IOException;
public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException;
ErosaConnection fork();
}

29
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java Normal file
View File

@@ -0,0 +1,29 @@
package com.alibaba.otter.canal.parse.inbound;
import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
/**
* 针对解析器提供一个多阶段协同的处理
*
* <pre>
* 1. 网络接收 (单线程)
* 2. 事件基本解析 (单线程，事件类型、DDL解析构造TableMeta、维护位点信息)
* 3. 事件深度解析 (多线程, DML事件数据的完整解析)
* 4. 投递到store (单线程)
* </pre>
*
* @author agapple 2018年7月3日 下午4:54:17
* @since 1.0.26
*/
public interface MultiStageCoprocessor extends CanalLifeCycle {
/**
* 网络数据投递
*/
public boolean publish(LogBuffer buffer);
public boolean publish(LogBuffer buffer, String binlogFileName);
public void reset();
}

22
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
View File

@@ -7,10 +7,12 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBBuilder;
@@ -90,8 +92,16 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
public void start() throws CanalParseException {
if (enableTsdb) {
if (tableMetaTSDB == null) {
// 初始化
tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
synchronized (CanalEventParser.class) {
try {
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
System.setProperty("canal.instance.destination", destination);
// 初始化
tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
} finally {
System.setProperty("canal.instance.destination", "");
}
}
}
}
@@ -117,6 +127,14 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
}
}
protected MultiStageCoprocessor buildMultiStageCoprocessor() {
return new MysqlMultiStageCoprocessor(parallelBufferSize,
parallelThreadSize,
(LogEventConvert) binlogParser,
transactionBuffer,
destination);
}
// ============================ setter / getter =========================
public void setConnectionCharsetNumber(byte connectionCharsetNumber) {

130
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
View File

@@ -4,16 +4,18 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.SinkFunction;
import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue;
import com.taobao.tddl.dbsync.binlog.FileLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
@@ -207,6 +209,132 @@ public class LocalBinLogConnection implements ErosaConnection {
throw new NotImplementedException();
}
@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
File current = new File(directory, binlogfilename);
FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
try {
fetcher.open(current, binlogPosition);
while (running) {
boolean needContinue = true;
while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate();
fetcher.consume(fetcher.limit());
// set filename
if (!coprocessor.publish(buffer, binlogfilename)) {
needContinue = false;
break;
}
}
if (needContinue) {// 读取下一个
fetcher.close(); // 关闭上一个文件
File nextFile;
if (needWait) {
nextFile = binlogs.waitForNextFile(current);
} else {
nextFile = binlogs.getNextFile(current);
}
if (nextFile == null) {
break;
}
current = nextFile;
fetcher.open(current);
binlogfilename = nextFile.getName();
} else {
break;// 跳出
}
}
} catch (InterruptedException e) {
logger.warn("LocalBinLogConnection dump interrupted");
} finally {
if (fetcher != null) {
fetcher.close();
}
}
}
@Override
public void dump(long timestampMills, MultiStageCoprocessor coprocessor) throws IOException {
List<File> currentBinlogs = binlogs.currentBinlogs();
File current = currentBinlogs.get(currentBinlogs.size() - 1);
long timestampSeconds = timestampMills / 1000;
String binlogFilename = null;
long binlogFileOffset = 0;
FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
LogDecoder decoder = new LogDecoder();
decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
decoder.handle(LogEvent.QUERY_EVENT);
decoder.handle(LogEvent.XID_EVENT);
LogContext context = new LogContext();
try {
fetcher.open(current);
context.setLogPosition(new LogPosition(current.getName()));
while (running) {
boolean needContinue = true;
String lastXidLogFilename = current.getName();
long lastXidLogFileOffset = 0;
binlogFilename = lastXidLogFilename;
binlogFileOffset = lastXidLogFileOffset;
while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context);
if (event != null) {
if (event.getWhen() > timestampSeconds) {
break;
}
needContinue = false;
if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
binlogFilename = lastXidLogFilename;
binlogFileOffset = lastXidLogFileOffset;
} else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
lastXidLogFilename = current.getName();
lastXidLogFileOffset = event.getLogPos();
}
} else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
lastXidLogFilename = current.getName();
lastXidLogFileOffset = event.getLogPos();
}
}
}
if (needContinue) {// 读取下一个
fetcher.close(); // 关闭上一个文件
File nextFile = binlogs.getBefore(current);
if (nextFile == null) {
break;
}
current = nextFile;
fetcher.open(current);
context.setLogPosition(new LogPosition(current.getName()));
} else {
break;// 跳出
}
}
} finally {
if (fetcher != null) {
fetcher.close();
}
}
dump(binlogFilename, binlogFileOffset, coprocessor);
}
@Override
public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException {
throw new NotImplementedException();
}
public ErosaConnection fork() {
LocalBinLogConnection connection = new LocalBinLogConnection();

84
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -8,8 +8,6 @@ import java.nio.charset.Charset;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpGTIDCommandPacket;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -17,8 +15,10 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpGTIDCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.SemiAckCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
@@ -26,9 +26,11 @@ import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.SinkFunction;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
@@ -156,7 +158,7 @@ public class MysqlConnection implements ErosaConnection {
}
if (event.getSemival() == 1) {
sendSemiAck(context.getLogPosition().getFileName(), binlogPosition);
sendSemiAck(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
}
}
}
@@ -167,20 +169,24 @@ public class MysqlConnection implements ErosaConnection {
sendBinlogDumpGTID(gtidSet);
DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
fetcher.start(connector.getChannel());
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
while (fetcher.fetch()) {
LogEvent event = null;
event = decoder.decode(fetcher, context);
try {
fetcher.start(connector.getChannel());
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
while (fetcher.fetch()) {
LogEvent event = null;
event = decoder.decode(fetcher, context);
if (event == null) {
throw new CanalParseException("parse failed");
}
if (event == null) {
throw new CanalParseException("parse failed");
}
if (!func.sink(event)) {
break;
if (!func.sink(event)) {
break;
}
}
} finally {
fetcher.close();
}
}
@@ -188,6 +194,53 @@ public class MysqlConnection implements ErosaConnection {
throw new NullPointerException("Not implement yet");
}
@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
updateSettings();
sendRegisterSlave();
sendBinlogDump(binlogfilename, binlogPosition);
((MysqlMultiStageCoprocessor) coprocessor).setConnection(this);
DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
try {
fetcher.start(connector.getChannel());
while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate();
fetcher.consume(fetcher.limit());
if (!coprocessor.publish(buffer)) {
break;
}
}
} finally {
fetcher.close();
}
}
@Override
public void dump(long timestamp, MultiStageCoprocessor coprocessor) throws IOException {
throw new NullPointerException("Not implement yet");
}
@Override
public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException {
updateSettings();
sendBinlogDumpGTID(gtidSet);
((MysqlMultiStageCoprocessor) coprocessor).setConnection(this);
DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
try {
fetcher.start(connector.getChannel());
while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate();
fetcher.consume(fetcher.limit());
if (!coprocessor.publish(buffer)) {
break;
}
}
} finally {
fetcher.close();
}
}
private void sendRegisterSlave() throws IOException {
RegisterSlaveCommandPacket cmd = new RegisterSlaveCommandPacket();
cmd.reportHost = authInfo.getAddress().getAddress().getHostAddress();
@@ -233,7 +286,7 @@ public class MysqlConnection implements ErosaConnection {
connector.setDumping(true);
}
private void sendSemiAck(String binlogfilename, Long binlogPosition) throws IOException {
public void sendSemiAck(String binlogfilename, Long binlogPosition) throws IOException {
SemiAckCommandPacket semiAckCmd = new SemiAckCommandPacket();
semiAckCmd.binlogFileName = binlogfilename;
semiAckCmd.binlogPosition = binlogPosition;
@@ -538,4 +591,5 @@ public class MysqlConnection implements ErosaConnection {
public void setAuthInfo(AuthenticationInfo authInfo) {
this.authInfo = authInfo;
}
}

5
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -653,6 +653,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
}
EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
if (isGTIDMode && fields.size() > 4) {
endPosition.setGtid(fields.get(4));
}
return endPosition;
} catch (IOException e) {
throw new CanalParseException("command : 'show master status' has an error!", e);
@@ -773,7 +776,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
}
if (StringUtils.equals(endPosition.getJournalName(), logfilename)
&& endPosition.getPosition() < logfileoffset) {
&& endPosition.getPosition() <= logfileoffset) {
return false;
}

430
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java Normal file
View File

@@ -0,0 +1,430 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.locks.LockSupport;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer;
import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.lmax.disruptor.BatchEventProcessor;
import com.lmax.disruptor.BlockingWaitStrategy;
import com.lmax.disruptor.EventFactory;
import com.lmax.disruptor.EventHandler;
import com.lmax.disruptor.ExceptionHandler;
import com.lmax.disruptor.InsufficientCapacityException;
import com.lmax.disruptor.LifecycleAware;
import com.lmax.disruptor.RingBuffer;
import com.lmax.disruptor.Sequence;
import com.lmax.disruptor.SequenceBarrier;
import com.lmax.disruptor.WorkHandler;
import com.lmax.disruptor.WorkerPool;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.LogPosition;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
/**
* 针对解析器提供一个多阶段协同的处理
*
* <pre>
* 1. 网络接收 (单线程)
* 2. 事件基本解析 (单线程，事件类型、DDL解析构造TableMeta、维护位点信息)
* 3. 事件深度解析 (多线程, DML事件数据的完整解析)
* 4. 投递到store (单线程)
* </pre>
*
* @author agapple 2018年7月3日 下午4:54:17
* @since 1.0.26
*/
public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implements MultiStageCoprocessor {
private LogEventConvert logEventConvert;
private EventTransactionBuffer transactionBuffer;
private ErosaConnection connection;
private int parserThreadCount;
private int ringBufferSize;
private RingBuffer<MessageEvent> disruptorMsgBuffer;
private ExecutorService parserExecutor;
private ExecutorService stageExecutor;
private String destination;
private volatile CanalParseException exception;
public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
EventTransactionBuffer transactionBuffer, String destination){
this.ringBufferSize = ringBufferSize;
this.parserThreadCount = parserThreadCount;
this.logEventConvert = logEventConvert;
this.transactionBuffer = transactionBuffer;
this.destination = destination;
}
@Override
public void start() {
super.start();
this.exception = null;
this.disruptorMsgBuffer = RingBuffer.createSingleProducer(new MessageEventFactory(),
ringBufferSize,
new BlockingWaitStrategy());
this.parserExecutor = Executors.newFixedThreadPool(parserThreadCount,
new NamedThreadFactory("MultiStageCoprocessor-Parser-" + destination));
this.stageExecutor = Executors.newFixedThreadPool(2, new NamedThreadFactory("MultiStageCoprocessor-other-"
+ destination));
SequenceBarrier sequenceBarrier = disruptorMsgBuffer.newBarrier();
ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
// stage 2
BatchEventProcessor<MessageEvent> simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
sequenceBarrier,
new SimpleParserStage());
simpleParserStage.setExceptionHandler(exceptionHandler);
disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence());
// stage 3
SequenceBarrier dmlParserSequenceBarrier = disruptorMsgBuffer.newBarrier(simpleParserStage.getSequence());
WorkHandler<MessageEvent>[] workHandlers = new DmlParserStage[parserThreadCount];
for (int i = 0; i < parserThreadCount; i++) {
workHandlers[i] = new DmlParserStage();
}
WorkerPool<MessageEvent> workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer,
dmlParserSequenceBarrier,
exceptionHandler,
workHandlers);
Sequence[] sequence = workerPool.getWorkerSequences();
disruptorMsgBuffer.addGatingSequences(sequence);
// stage 4
SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence);
BatchEventProcessor<MessageEvent> sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
sinkSequenceBarrier,
new SinkStoreStage());
sinkStoreStage.setExceptionHandler(exceptionHandler);
disruptorMsgBuffer.addGatingSequences(sinkStoreStage.getSequence());
// start work
stageExecutor.submit(simpleParserStage);
stageExecutor.submit(sinkStoreStage);
workerPool.start(parserExecutor);
}
@Override
public void stop() {
try {
parserExecutor.shutdownNow();
} catch (Throwable e) {
// ignore
}
try {
stageExecutor.shutdownNow();
} catch (Throwable e) {
// ignore
}
super.stop();
}
/**
* 网络数据投递
*/
public boolean publish(LogBuffer buffer) {
return publish(buffer, null);
}
public boolean publish(LogBuffer buffer, String binlogFileName) {
if (!isStart()) {
if (exception != null) {
throw exception;
}
return false;
}
/**
* 由于改为processor仅终止自身stage而不是stop，那么需要由incident标识coprocessor是否正常工作。
* 让dump线程能够及时感知
*/
if (exception != null) {
throw exception;
}
boolean interupted = false;
do {
try {
long next = disruptorMsgBuffer.tryNext();
MessageEvent event = disruptorMsgBuffer.get(next);
event.setBuffer(buffer);
if (binlogFileName != null) {
event.setBinlogFileName(binlogFileName);
}
disruptorMsgBuffer.publish(next);
break;
} catch (InsufficientCapacityException e) {
// park
LockSupport.parkNanos(1L);
interupted = Thread.interrupted();
}
} while (!interupted && isStart());
return isStart();
}
@Override
public void reset() {
if (isStart()) {
stop();
}
start();
}
private class SimpleParserStage implements EventHandler<MessageEvent>, LifecycleAware {
private LogDecoder decoder;
private LogContext context;
public SimpleParserStage(){
decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
context = new LogContext();
}
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
try {
LogBuffer buffer = event.getBuffer();
if (StringUtils.isNotEmpty(event.getBinlogFileName())
&& !context.getLogPosition().getFileName().equals(event.getBinlogFileName())) {
// set roate binlog file name
context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
.getPosition()));
}
LogEvent logEvent = decoder.decode(buffer, context);
event.setEvent(logEvent);
int eventType = logEvent.getHeader().getType();
TableMeta tableMeta = null;
boolean needDmlParse = false;
switch (eventType) {
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
tableMeta = logEventConvert.parseRowsEventForTableMeta((WriteRowsLogEvent) logEvent);
needDmlParse = true;
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
needDmlParse = true;
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
tableMeta = logEventConvert.parseRowsEventForTableMeta((DeleteRowsLogEvent) logEvent);
needDmlParse = true;
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
needDmlParse = true;
break;
default:
CanalEntry.Entry entry = logEventConvert.parse(event.getEvent(), false);
event.setEntry(entry);
}
// 记录一下DML的表结构
event.setNeedDmlParse(needDmlParse);
event.setTable(tableMeta);
} catch (Throwable e) {
exception = new CanalParseException(e);
throw exception;
}
}
@Override
public void onStart() {
}
@Override
public void onShutdown() {
}
}
private class DmlParserStage implements WorkHandler<MessageEvent>, LifecycleAware {
@Override
public void onEvent(MessageEvent event) throws Exception {
try {
if (event.isNeedDmlParse()) {
int eventType = event.getEvent().getHeader().getType();
CanalEntry.Entry entry = null;
switch (eventType) {
case LogEvent.ROWS_QUERY_LOG_EVENT:
entry = logEventConvert.parse(event.getEvent(), false);
break;
default:
// 单独解析dml事件
entry = logEventConvert.parseRowsEvent((RowsLogEvent) event.getEvent(), event.getTable());
}
event.setEntry(entry);
}
} catch (Throwable e) {
exception = new CanalParseException(e);
throw exception;
}
}
@Override
public void onStart() {
}
@Override
public void onShutdown() {
}
}
private class SinkStoreStage implements EventHandler<MessageEvent>, LifecycleAware {
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
try {
if (event.getEntry() != null) {
transactionBuffer.add(event.getEntry());
}
LogEvent logEvent = event.getEvent();
if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
// semi ack回报
((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
logEvent.getHeader().getLogPos());
}
// clear for gc
event.setBuffer(null);
event.setBinlogFileName(null);
event.setEvent(null);
event.setTable(null);
event.setEntry(null);
event.setNeedDmlParse(false);
} catch (Throwable e) {
exception = new CanalParseException(e);
throw exception;
}
}
@Override
public void onStart() {
}
@Override
public void onShutdown() {
}
}
class MessageEvent {
private String binlogFileName; // for local binlog parse
private LogBuffer buffer;
private CanalEntry.Entry entry;
private boolean needDmlParse = false;
private TableMeta table;
private LogEvent event;
public String getBinlogFileName() {
return binlogFileName;
}
public void setBinlogFileName(String binlogFileName) {
this.binlogFileName = binlogFileName;
}
public LogBuffer getBuffer() {
return buffer;
}
public void setBuffer(LogBuffer buffer) {
this.buffer = buffer;
}
public LogEvent getEvent() {
return event;
}
public void setEvent(LogEvent event) {
this.event = event;
}
public CanalEntry.Entry getEntry() {
return entry;
}
public void setEntry(CanalEntry.Entry entry) {
this.entry = entry;
}
public boolean isNeedDmlParse() {
return needDmlParse;
}
public void setNeedDmlParse(boolean needDmlParse) {
this.needDmlParse = needDmlParse;
}
public TableMeta getTable() {
return table;
}
public void setTable(TableMeta table) {
this.table = table;
}
}
class SimpleFatalExceptionHandler implements ExceptionHandler {
@Override
public void handleEventException(final Throwable ex, final long sequence, final Object event) {
}
@Override
public void handleOnStartException(final Throwable ex) {
}
@Override
public void handleOnShutdownException(final Throwable ex) {
}
}
class MessageEventFactory implements EventFactory<MessageEvent> {
public MessageEvent newInstance() {
return new MessageEvent();
}
}
public void setLogEventConvert(LogEventConvert logEventConvert) {
this.logEventConvert = logEventConvert;
}
public void setTransactionBuffer(EventTransactionBuffer transactionBuffer) {
this.transactionBuffer = transactionBuffer;
}
public void setConnection(ErosaConnection connection) {
this.connection = connection;
}
}

32
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java
View File

@@ -19,30 +19,30 @@ import com.taobao.tddl.dbsync.binlog.LogFetcher;
*/
public class DirectLogFetcher extends LogFetcher {
protected static final Logger logger = LoggerFactory.getLogger(DirectLogFetcher.class);
protected static final Logger logger = LoggerFactory.getLogger(DirectLogFetcher.class);
// Master heartbeat interval
public static final int MASTER_HEARTBEAT_PERIOD_SECONDS = 15;
public static final int MASTER_HEARTBEAT_PERIOD_SECONDS = 15;
// +10s 确保 timeout > heartbeat interval
private static final int READ_TIMEOUT_MILLISECONDS = (MASTER_HEARTBEAT_PERIOD_SECONDS + 10) * 1000;
private static final int READ_TIMEOUT_MILLISECONDS = (MASTER_HEARTBEAT_PERIOD_SECONDS + 10) * 1000;
/** Command to dump binlog */
public static final byte COM_BINLOG_DUMP = 18;
public static final byte COM_BINLOG_DUMP = 18;
/** Packet header sizes */
public static final int NET_HEADER_SIZE = 4;
public static final int SQLSTATE_LENGTH = 5;
public static final int NET_HEADER_SIZE = 4;
public static final int SQLSTATE_LENGTH = 5;
/** Packet offsets */
public static final int PACKET_LEN_OFFSET = 0;
public static final int PACKET_SEQ_OFFSET = 3;
public static final int PACKET_LEN_OFFSET = 0;
public static final int PACKET_SEQ_OFFSET = 3;
/** Maximum packet length */
public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
private SocketChannel channel;
private boolean issemi = false;
private boolean issemi = false;
// private BufferedInputStream input;
@@ -64,9 +64,6 @@ public class DirectLogFetcher extends LogFetcher {
if ("1".equals(dbsemi)) {
issemi = true;
}
// 和mysql driver一样，提供buffer机制，提升读取binlog速度
// this.input = new
// BufferedInputStream(channel.socket().getInputStream(), 16384);
}
/**
@@ -171,10 +168,13 @@ public class DirectLogFetcher extends LogFetcher {
private final boolean fetch0(final int off, final int len) throws IOException {
ensureCapacity(off + len);
byte[] read = channel.read(len, READ_TIMEOUT_MILLISECONDS);
System.arraycopy(read, 0, this.buffer, off, len);
// byte[] read = channel.read(len, READ_TIMEOUT_MILLISECONDS);
// System.arraycopy(read, 0, this.buffer, off, len);
if (limit < off + len) limit = off + len;
channel.read(buffer, off, len, READ_TIMEOUT_MILLISECONDS);
if (limit < off + len) {
limit = off + len;
}
return true;
}

181
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -49,7 +49,6 @@ import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
@@ -91,8 +90,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时
private volatile AviaterRegexFilter nameBlackFilter;
private TableMetaCacheInterface tableMetaCache;
private String binlogFileName = "mysql-bin.000001";
private Charset charset = Charset.defaultCharset();
private boolean filterQueryDcl = false;
private boolean filterQueryDml = false;
@@ -122,9 +123,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
int eventType = logEvent.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
binlogFileName = ((RotateLogEvent) logEvent).getFilename();
break;
case LogEvent.QUERY_EVENT:
return parseQueryEvent((QueryLogEvent) logEvent, isSeek);
case LogEvent.XID_EVENT:
@@ -161,7 +159,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
public void reset() {
// do nothing
binlogFileName = "mysql-bin.000001";
if (tableMetaCache != null) {
tableMetaCache.clearTableMeta();
}
@@ -176,8 +173,14 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (gtidSet != null) {
gtidSet.update(value);
}
if (logEvent.getLastCommitted() != null) {
builder.setKey("lastCommitted");
builder.setValue(String.valueOf(logEvent.getLastCommitted()));
builder.setKey("sequenceNumber");
builder.setValue(String.valueOf(logEvent.getSequenceNumber()));
}
Header header = createHeader("", logHeader, "", "", EventType.GTID);
Header header = createHeader(logHeader, "", "", EventType.GTID);
return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
@@ -190,7 +193,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
beginBuilder.addProps(createSpecialPair(XA_TYPE, XA_START));
beginBuilder.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_START)));
TransactionBegin transactionBegin = beginBuilder.build();
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
} else if (StringUtils.startsWithIgnoreCase(queryString, XA_END)) {
// xa start use TransactionEnd
@@ -199,11 +202,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
endBuilder.addProps(createSpecialPair(XA_TYPE, XA_END));
endBuilder.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_END)));
TransactionEnd transactionEnd = endBuilder.build();
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
} else if (StringUtils.startsWithIgnoreCase(queryString, XA_COMMIT)) {
// xa commit
Header header = createHeader(binlogFileName, event.getHeader(), "", "", EventType.XACOMMIT);
Header header = createHeader(event.getHeader(), "", "", EventType.XACOMMIT);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setSql(queryString);
rowChangeBuider.addProps(createSpecialPair(XA_TYPE, XA_COMMIT));
@@ -212,7 +215,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
} else if (StringUtils.startsWithIgnoreCase(queryString, XA_ROLLBACK)) {
// xa rollback
Header header = createHeader(binlogFileName, event.getHeader(), "", "", EventType.XAROLLBACK);
Header header = createHeader(event.getHeader(), "", "", EventType.XAROLLBACK);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setSql(queryString);
rowChangeBuider.addProps(createSpecialPair(XA_TYPE, XA_ROLLBACK));
@@ -221,11 +224,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
} else if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
} else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
TransactionEnd transactionEnd = createTransactionEnd(0L); // MyISAM可能不会有xid事件
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
} else {
boolean notFilter = false;
@@ -269,9 +272,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
tableMetaCache.apply(position, fulltbName, queryString, null);
}
Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
Header header = createHeader(event.getHeader(), schemaName, tableName, type);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
if (type != EventType.QUERY) {
if (type != EventType.QUERY && type != EventType.INSERT && type != EventType.UPDATE
&& type != EventType.DELETE) {
rowChangeBuider.setIsDdl(true);
}
rowChangeBuider.setSql(queryString);
@@ -428,52 +432,78 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private Entry parseXidEvent(XidLogEvent event) {
TransactionEnd transactionEnd = createTransactionEnd(event.getXid());
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
}
private Entry parseRowsEvent(RowsLogEvent event) {
public TableMeta parseRowsEventForTableMeta(RowsLogEvent event) {
TableMapLogEvent table = event.getTable();
if (table == null) {
// tableId对应的记录不存在
throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
}
boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName());
boolean isRDSHeartBeat = tableMetaCache.isOnRDS() && isRDSHeartBeat(table.getDbName(), table.getTableName());
String fullname = table.getDbName() + "." + table.getTableName();
// check name filter
if (nameFilter != null && !nameFilter.filter(fullname)) {
return null;
}
if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
return null;
}
// if (isHeartBeat || isRDSHeartBeat) {
// // 忽略rds模式的mysql.ha_health_check心跳数据
// return null;
// }
TableMeta tableMeta = null;
if (isRDSHeartBeat) {
// 处理rds模式的mysql.ha_health_check心跳数据
// 主要RDS的心跳表基本无权限,需要mock一个tableMeta
FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0");
FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0");
tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
} else if (isHeartBeat) {
// 处理alisql模式的test.heartbeat心跳数据
// 心跳表基本无权限,需要mock一个tableMeta
FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null);
FieldMeta typeMeta = new FieldMeta("ts", "int(11)", true, false, null);
tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
}
EntryPosition position = createPosition(event.getHeader());
if (tableMetaCache != null && tableMeta == null) {// 入错存在table meta
tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position);
if (tableMeta == null) {
if (!filterTableError) {
throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
}
}
}
return tableMeta;
}
public Entry parseRowsEvent(RowsLogEvent event) {
return parseRowsEvent(event, null);
}
public Entry parseRowsEvent(RowsLogEvent event, TableMeta tableMeta) {
if (filterRows) {
return null;
}
try {
TableMapLogEvent table = event.getTable();
if (table == null) {
// tableId对应的记录不存在
throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
if (tableMeta == null) { // 如果没有外部指定
tableMeta = parseRowsEventForTableMeta(event);
}
boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName());
boolean isRDSHeartBeat = tableMetaCache.isOnRDS()
&& isRDSHeartBeat(table.getDbName(), table.getTableName());
String fullname = table.getDbName() + "." + table.getTableName();
// check name filter
if (nameFilter != null && !nameFilter.filter(fullname)) {
if (tableMeta == null) {
// 拿不到表结构,执行忽略
return null;
}
if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
return null;
}
// if (isHeartBeat || isRDSHeartBeat) {
// // 忽略rds模式的mysql.ha_health_check心跳数据
// return null;
// }
TableMeta tableMeta = null;
if (isRDSHeartBeat) {
// 处理rds模式的mysql.ha_health_check心跳数据
// 主要RDS的心跳表基本无权限,需要mock一个tableMeta
FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0");
FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0");
tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
} else if (isHeartBeat) {
// 处理alisql模式的test.heartbeat心跳数据
// 心跳表基本无权限,需要mock一个tableMeta
FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null);
FieldMeta typeMeta = new FieldMeta("type", "int(11)", true, false, null);
tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
}
EventType eventType = null;
int type = event.getHeader().getType();
@@ -487,12 +517,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
}
Header header = createHeader(binlogFileName,
event.getHeader(),
table.getDbName(),
table.getTableName(),
eventType);
EntryPosition position = createPosition(event.getHeader());
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setTableId(event.getTableId());
rowChangeBuider.setIsDdl(false);
@@ -501,18 +525,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
boolean tableError = false;
if (tableMetaCache != null && tableMeta == null) {// 入错存在table meta
// cache
tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position);
if (tableMeta == null) {
tableError = true;
if (!filterTableError) {
throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
}
}
}
boolean tableError = false;
int rowsCount = 0;
while (buffer.nextOneRow(columns)) {
// 处理row记录
RowData.Builder rowDataBuilder = RowData.newBuilder();
@@ -533,16 +548,20 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
}
rowsCount ++;
rowChangeBuider.addRowDatas(rowDataBuilder.build());
}
TableMapLogEvent table = event.getTable();
Header header = createHeader(event.getHeader(), table.getDbName(), table.getTableName(), eventType, rowsCount);
RowChange rowChange = rowChangeBuider.build();
if (tableError) {
Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
return null;
} else {
Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
Entry entry = createEntry(header, EntryType.ROWDATA, rowChange.toByteString());
return entry;
}
} catch (Exception e) {
@@ -551,7 +570,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
private EntryPosition createPosition(LogHeader logHeader) {
return new EntryPosition(binlogFileName,
return new EntryPosition(logHeader.getLogFileName(),
logHeader.getLogPos(),
logHeader.getWhen() * 1000L,
logHeader.getServerId()); // 记录到秒
@@ -638,11 +657,14 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
// fixed issue
// https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
boolean isBinary = false;
boolean isSingleBit = false;
if (fieldMeta != null) {
if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
isBinary = true;
} else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
isBinary = true;
} else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "TINYINT(1)")) {
isSingleBit = true;
}
}
buffer.nextValue(info.type, info.meta, isBinary);
@@ -652,6 +674,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
int javaType = buffer.getJavaType();
if (isSingleBit && javaType == Types.TINYINT) {
javaType = Types.BIT;
}
if (buffer.isNull()) {
columnBuilder.setIsNull(true);
} else {
@@ -768,7 +793,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
private Entry buildQueryEntry(String queryString, LogHeader logHeader, String tableName) {
Header header = createHeader(binlogFileName, logHeader, "", tableName, EventType.QUERY);
Header header = createHeader(logHeader, "", tableName, EventType.QUERY);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setSql(queryString);
rowChangeBuider.setEventType(EventType.QUERY);
@@ -776,19 +801,23 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
Header header = createHeader(logHeader, "", "", EventType.QUERY);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setSql(queryString);
rowChangeBuider.setEventType(EventType.QUERY);
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
private Header createHeader(String binlogFile, LogHeader logHeader, String schemaName, String tableName,
EventType eventType) {
private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType) {
return createHeader(logHeader, schemaName, tableName, eventType, -1);
}
private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType, Integer rowsCount) {
// header会做信息冗余,方便以后做检索或者过滤
Header.Builder headerBuilder = Header.newBuilder();
headerBuilder.setVersion(version);
headerBuilder.setLogfileName(binlogFile);
headerBuilder.setLogfileName(logHeader.getLogFileName());
headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
headerBuilder.setServerId(logHeader.getServerId());
headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
@@ -809,6 +838,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
String gtid = gtidSet.toString();
headerBuilder.setGtid(gtid);
}
// add rowsCount suppport
if (rowsCount > 0) {
Pair pair = createSpecialPair("rowsCount", String.valueOf(rowsCount));
headerBuilder.addProps(pair);
}
return headerBuilder.build();
}

18
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java
View File

@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import com.alibaba.fastsql.sql.SQLUtils;
import com.alibaba.fastsql.sql.ast.SQLExpr;
import com.alibaba.fastsql.sql.ast.SQLStatement;
@@ -18,9 +20,11 @@ import com.alibaba.fastsql.sql.ast.statement.SQLAlterTableItem;
import com.alibaba.fastsql.sql.ast.statement.SQLAlterTableRename;
import com.alibaba.fastsql.sql.ast.statement.SQLAlterTableStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLConstraint;
import com.alibaba.fastsql.sql.ast.statement.SQLCreateDatabaseStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLCreateIndexStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLDeleteStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLDropDatabaseStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLDropIndexStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLDropTableStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
@@ -157,6 +161,17 @@ public class DruidDdlParser {
processName(ddlResult, schmeaName, delete.getTableName(), false);
ddlResult.setType(EventType.DELETE);
ddlResults.add(ddlResult);
} else if (statement instanceof SQLCreateDatabaseStatement) {
DdlResult ddlResult = new DdlResult();
ddlResult.setType(EventType.QUERY);
// 只设置schema
processName(ddlResult, schmeaName, null, false);
ddlResults.add(ddlResult);
} else if (statement instanceof SQLDropDatabaseStatement) {
DdlResult ddlResult = new DdlResult();
ddlResult.setType(EventType.QUERY);
// 只设置schema
processName(ddlResult, schmeaName, null, false);
}
}
@@ -165,6 +180,9 @@ public class DruidDdlParser {
private static void processName(DdlResult ddlResult, String schema, SQLExpr sqlName, boolean isOri) {
if (sqlName == null) {
if (StringUtils.isNotBlank(schema)) {
ddlResult.setSchemaName(schema);
}
return;
}

27
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -70,7 +70,9 @@ public class DatabaseTableMeta implements TableMetaTSDB {
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "[scheduler-table-meta-snapshot]");
Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]");
thread.setDaemon(true);
return thread;
}
});
@@ -298,13 +300,24 @@ public class DatabaseTableMeta implements TableMetaTSDB {
createDDL = packet.getFieldValues().get(1);
tableMetaFromDB.setFields(TableMetaCache.parseTableMeta(schema, table, packet));
}
} catch (IOException e) {
if (e.getMessage().contains("errorNumber=1146")) {
logger.error("table not exist in db , pls check :" + getFullName(schema, table) + " , mem : "
+ tableMetaFromMem);
return false;
} catch (Throwable e) {
try {
// retry for broke pipe, see:
// https://github.com/alibaba/canal/issues/724
connection.reconnect();
ResultSetPacket packet = connection.query("show create table " + getFullName(schema, table));
if (packet.getFieldValues().size() > 1) {
createDDL = packet.getFieldValues().get(1);
tableMetaFromDB.setFields(TableMetaCache.parseTableMeta(schema, table, packet));
}
} catch (IOException e1) {
if (e.getMessage().contains("errorNumber=1146")) {
logger.error("table not exist in db , pls check :" + getFullName(schema, table) + " , mem : "
+ tableMetaFromMem);
return false;
}
throw new CanalParseException(e);
}
throw new CanalParseException(e);
}
boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -94,7 +94,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
tableMeta = tableMetas.get(keys);
if (tableMeta == null) {
Schema schemaRep = repository.findSchema(schema);
if (schema == null) {
if (schemaRep == null) {
return null;
}
SchemaObject data = schemaRep.findTable(table);

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDO.java
View File

@@ -87,7 +87,6 @@ public class MetaHistoryDO {
this.useSchema = useSchema;
}
public String getExtra() {
return extra;
}

89
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java Normal file
View File

@@ -0,0 +1,89 @@
package com.alibaba.otter.canal.parse;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class MysqlBinlogDumpPerformanceTest {
public static void main(String args[]) {
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.001699", 120L, 100L);
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setFilterQueryDml(true);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3328), "root", "hello"));
controller.setMasterPosition(startPosition);
controller.setEnableTsdb(false);
controller.setDestination("example");
controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
// controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
// controller.setEventBlackFilter(new
// AviaterRegexFilter("canal_tsdb\\..*"));
controller.setParallel(true);
controller.setParallelBufferSize(256);
controller.setParallelThreadSize(16);
controller.setIsGTIDMode(false);
final AtomicLong sum = new AtomicLong(0);
final AtomicLong last = new AtomicLong(0);
final AtomicLong start = new AtomicLong(System.currentTimeMillis());
final AtomicLong end = new AtomicLong(0);
controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException,
InterruptedException {
sum.addAndGet(entrys.size());
long current = sum.get();
if (current - last.get() >= 100000) {
end.set(System.currentTimeMillis());
long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
last.set(current);
start.set(end.get());
}
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
}
});
controller.start();
try {
Thread.sleep(100 * 1000 * 1000L);
} catch (InterruptedException e) {
}
controller.stop();
}
public static abstract class AbstractCanalEventSinkTest<T> extends AbstractCanalLifeCycle implements CanalEventSink<T> {
public void interrupt() {
}
}
}

82
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java Normal file
View File

@@ -0,0 +1,82 @@
package com.alibaba.otter.canal.parse;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
public class MysqlBinlogEventPerformanceTest {
protected static Charset charset = Charset.forName("utf-8");
public static void main(String args[]) {
DirectLogFetcher fetcher = new DirectLogFetcher();
try {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
connector.connect();
updateSettings(connector);
sendBinlogDump(connector, "mysql-bin.000006", 120L, 3);
fetcher.start(connector.getChannel());
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
AtomicLong sum = new AtomicLong(0);
long start = System.currentTimeMillis();
long last = 0;
long end = 0;
while (fetcher.fetch()) {
decoder.decode(fetcher, context);
sum.incrementAndGet();
long current = sum.get();
if (current - last >= 100000) {
end = System.currentTimeMillis();
long tps = ((current - last) * 1000) / (end - start);
System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
last = current;
start = end;
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
fetcher.close();
} catch (IOException e) {
}
}
}
private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
throws IOException {
BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
binlogDumpCmd.binlogFileName = binlogfilename;
binlogDumpCmd.binlogPosition = binlogPosition;
binlogDumpCmd.slaveServerId = slaveId;
byte[] cmdBody = binlogDumpCmd.toBytes();
HeaderPacket binlogDumpHeader = new HeaderPacket();
binlogDumpHeader.setPacketBodyLength(cmdBody.length);
binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody);
}
private static void updateSettings(MysqlConnector connector) throws IOException {
update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector);
}
public static void update(String cmd, MysqlConnector connector) throws IOException {
MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
exector.update(cmd);
}
}

192
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java Normal file
View File

@@ -0,0 +1,192 @@
package com.alibaba.otter.canal.parse;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.BitSet;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
public class MysqlBinlogParsePerformanceTest {
protected static Charset charset = Charset.forName("utf-8");
public static void main(String args[]) {
DirectLogFetcher fetcher = new DirectLogFetcher();
try {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
connector.connect();
updateSettings(connector);
sendBinlogDump(connector, "mysql-bin.000006", 120L, 3);
fetcher.start(connector.getChannel());
final BlockingQueue<LogBuffer> buffer = new ArrayBlockingQueue<LogBuffer>(1024);
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
consumer(buffer);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
thread.start();
while (fetcher.fetch()) {
buffer.put(fetcher.duplicate());
fetcher.consume(fetcher.limit());
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
fetcher.close();
} catch (IOException e) {
}
}
}
public static void consumer(BlockingQueue<LogBuffer> buffer) throws IOException, InterruptedException {
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
AtomicLong sum = new AtomicLong(0);
long start = System.currentTimeMillis();
long last = 0;
long end = 0;
while (true) {
LogEvent event = null;
event = decoder.decode(buffer.take(), context);
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
parseRowsEvent((WriteRowsLogEvent) event, sum);
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
parseRowsEvent((UpdateRowsLogEvent) event, sum);
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
parseRowsEvent((DeleteRowsLogEvent) event, sum);
break;
case LogEvent.XID_EVENT:
sum.incrementAndGet();
break;
case LogEvent.QUERY_EVENT:
sum.incrementAndGet();
break;
default:
break;
}
long current = sum.get();
if (current - last >= 100000) {
end = System.currentTimeMillis();
long tps = ((current - last) * 1000) / (end - start);
System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
last = current;
start = end;
}
}
}
private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
throws IOException {
BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
binlogDumpCmd.binlogFileName = binlogfilename;
binlogDumpCmd.binlogPosition = binlogPosition;
binlogDumpCmd.slaveServerId = slaveId;
byte[] cmdBody = binlogDumpCmd.toBytes();
HeaderPacket binlogDumpHeader = new HeaderPacket();
binlogDumpHeader.setPacketBodyLength(cmdBody.length);
binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody);
}
private static void updateSettings(MysqlConnector connector) throws IOException {
update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector);
}
public static void update(String cmd, MysqlConnector connector) throws IOException {
MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
exector.update(cmd);
}
public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
try {
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
while (buffer.nextOneRow(columns)) {
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
parseOneRow(event, buffer, columns, true);
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
parseOneRow(event, buffer, columns, false);
} else {
parseOneRow(event, buffer, columns, false);
if (!buffer.nextOneRow(changeColumns)) {
break;
}
parseOneRow(event, buffer, changeColumns, true);
}
sum.incrementAndGet();
}
} catch (Exception e) {
throw new RuntimeException("parse row data failed.", e);
}
}
public static void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
throws UnsupportedEncodingException {
TableMapLogEvent map = event.getTable();
if (map == null) {
throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
}
final int columnCnt = map.getColumnCnt();
final ColumnInfo[] columnInfo = map.getColumnInfo();
for (int i = 0; i < columnCnt; i++) {
if (!cols.get(i)) {
continue;
}
ColumnInfo info = columnInfo[i];
buffer.nextValue(info.type, info.meta);
if (buffer.isNull()) {
} else {
buffer.getValue();
}
}
}
}

42
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java
View File

@@ -7,7 +7,6 @@ import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.store.CanalEventStore;
@@ -18,7 +17,7 @@ import com.alibaba.otter.canal.store.model.Events;
public class DummyEventStore implements CanalEventStore<Event> {
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static final String messgae = "{0} [{1}:{2}:{3}] {4} {5}.{6}";
private static final String messgae = "{0} [{1}:{2}:{3}]";
public void ack(Position position) throws CanalStoreException {
@@ -82,46 +81,40 @@ public class DummyEventStore implements CanalEventStore<Event> {
public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
Date date = new Date(data.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
new Object[] { Thread.currentThread().getName(), data.getJournalName(),
String.valueOf(data.getPosition()), format.format(date) }));
}
}
}
public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
Date date = new Date(data.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
new Object[] { Thread.currentThread().getName(), data.getJournalName(),
String.valueOf(data.getPosition()), format.format(date) }));
}
}
return true;
@@ -131,23 +124,20 @@ public class DummyEventStore implements CanalEventStore<Event> {
System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
Date date = new Date(data.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
new Object[] { Thread.currentThread().getName(), data.getJournalName(),
String.valueOf(data.getPosition()), format.format(date) }));
}
}

3
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
View File

@@ -41,6 +41,9 @@ public class MysqlDumpTest {
controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml");
controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*"));
controller.setParallel(true);
controller.setParallelBufferSize(256);
controller.setParallelThreadSize(2);
controller.setIsGTIDMode(false);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

30
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java Normal file
View File

@@ -0,0 +1,30 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.junit.Test;
import com.alibaba.fastsql.sql.repository.SchemaObject;
import com.alibaba.fastsql.sql.repository.SchemaRepository;
import com.alibaba.fastsql.util.JdbcConstants;
/**
* @author agapple 2018年6月7日 下午5:36:13
* @since 3.1.9
*/
public class FastsqlSchemaTest {
@Test
public void testSimple() throws FileNotFoundException, IOException {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = "create table yushitai_test.card_record ( id bigint auto_increment) auto_increment=256 "
+ "; alter table yushitai_test.card_record add column customization_id bigint unsigned NOT NULL COMMENT 'TEST' ;"
+ "; rename table yushitai_test.card_record to yushitai_test._card_record_del;";
repository.console(sql);
repository.setDefaultSchema("yushitai_test");
SchemaObject table = repository.findTable("_card_record_del");
System.out.println(table.getStatement().toString());
}
}

40
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java Normal file
View File

@@ -0,0 +1,40 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.io.File;
import java.io.FileInputStream;
import java.net.URL;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
/**
* @author agapple 2017年8月1日 下午7:15:54
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" })
public class MemoryTableMeta_DDL_Test {
@Test
public void test1() throws Throwable {
MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
File create = new File(dummyFile.getParent() + "/ddl", "ddl_test1.sql");
String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
memoryTableMeta.apply(null, "test", sql, null);
TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
System.out.println(meta);
Assert.assertNotNull(meta.getFieldMetaByName("customization_id"));
meta = memoryTableMeta.find("yushitai_test", "_card_record_gho");
Assert.assertNull(meta);
}
}

21
parse/src/test/resources/ddl/ddl_test1.sql Normal file
View File

@@ -0,0 +1,21 @@
create table yushitai_test.card_record (
id bigint auto_increment,
last_update timestamp not null DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
hint varchar(64) charset ascii not null,
value varchar(255) charset ascii not null,
primary key(id),
unique key hint_uidx(hint)
) auto_increment=256 ;
DROP TABLE IF EXISTS _card_record_gho /* generated by server */ ;
DROP TABLE IF EXISTS _card_record_del /* generated by server */ ;
create /* gh-ost */ table yushitai_test._card_record_gho like yushitai_test.card_record ;
alter /* gh-ost */ table yushitai_test._card_record_gho add column customization_id bigint unsigned NOT NULL COMMENT 'TEST' ;
create /* gh-ost */ table yushitai_test._card_record_del (
id int auto_increment primary key
) engine=InnoDB comment='ghost-cut-over-sentry' ;
DROP TABLE IF EXISTS _card_record_del /* generated by server */ ;
rename /* gh-ost */ table yushitai_test.card_record to yushitai_test._card_record_del, yushitai_test._card_record_gho to yushitai_test.card_record;