chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java Normal file
View File

@@ -0,0 +1,101 @@
package com.alibaba.otter.canal.parse;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
public class DirectLogFetcherTest {
@Test
public void testSimple() {
DirectLogFetcher fetcher = new DirectLogFetcher();
try {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
connector.connect();
sendBinlogDump(connector, "mysql-bin.001016", 4L, 3);
fetcher.start(connector.getChannel());
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
while (fetcher.fetch()) {
LogEvent event = null;
event = decoder.decode(fetcher, context);
if (event == null) {
throw new RuntimeException("parse failed");
}
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
// binlogFileName = ((RotateLogEvent)
// event).getFilename();
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
// parseRowsEvent((WriteRowsLogEvent) event);
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
// parseRowsEvent((UpdateRowsLogEvent) event);
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
// parseRowsEvent((DeleteRowsLogEvent) event);
break;
case LogEvent.QUERY_EVENT:
// parseQueryEvent((QueryLogEvent) event);
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
// parseRowsQueryEvent((RowsQueryLogEvent) event);
break;
case LogEvent.ANNOTATE_ROWS_EVENT:
break;
case LogEvent.XID_EVENT:
break;
default:
break;
}
}
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
} finally {
try {
fetcher.close();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
private void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
throws IOException {
BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
binlogDumpCmd.binlogFileName = binlogfilename;
binlogDumpCmd.binlogPosition = binlogPosition;
binlogDumpCmd.slaveServerId = slaveId;
byte[] cmdBody = binlogDumpCmd.toBytes();
HeaderPacket binlogDumpHeader = new HeaderPacket();
binlogDumpHeader.setPacketBodyLength(cmdBody.length);
binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
PacketManager.write(connector.getChannel(), new ByteBuffer[] { ByteBuffer.wrap(binlogDumpHeader.toBytes()),
ByteBuffer.wrap(cmdBody) });
}
}

59
parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java Normal file
View File

@@ -0,0 +1,59 @@
package com.alibaba.otter.canal.parse.helper;
/**
* 用于检查超时, 主要用于启动服务以后，如果在指定的时间内没有响应，则自动退出
*
* @author: yuanzu Date: 12-9-26 Time: 上午10:55
*/
public class TimeoutChecker {
/**
* 最后一次动作的时间
*/
private long lastTouch;
/**
* 超时时间
*/
private long timeoutMillis;
/**
* 运行标志
*/
private boolean running = true;
/**
* default 3s
*/
private static final long DEFAULT_TIMEOUT_MILLIS = 3 * 1000;
public TimeoutChecker(long timeoutMillis){
this.timeoutMillis = timeoutMillis;
touch();
}
public TimeoutChecker(){
this(DEFAULT_TIMEOUT_MILLIS);
}
/**
* 更新
*/
public void touch() {
this.lastTouch = System.currentTimeMillis();
}
/**
* 等待空闲
*
* @throws InterruptedException
*/
public void waitForIdle() throws InterruptedException {
while (this.running && (System.currentTimeMillis() - this.lastTouch) < this.timeoutMillis) {
Thread.sleep(50);
}
}
public void stop() {
this.running = false;
}
}

138
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java Normal file
View File

@@ -0,0 +1,138 @@
package com.alibaba.otter.canal.parse.inbound;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.Header;
public class EventTransactionBufferTest {
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static final String messgae = "{0} [{1}:{2}:{3}] {4}.{5}";
@Test
public void testTransactionFlush() {
final int bufferSize = 64;
final int transactionSize = 5;
EventTransactionBuffer buffer = new EventTransactionBuffer();
buffer.setBufferSize(bufferSize);
buffer.setFlushCallback(new TransactionFlushCallback() {
public void flush(List<Entry> transaction) throws InterruptedException {
Assert.assertEquals(transactionSize, transaction.size());
System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
for (Entry data : transaction) {
CanalEntry.Header header = data.getHeader();
Date date = new Date(header.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntryType() == EntryType.TRANSACTIONEND) {
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae, new Object[] {
Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(),
format.format(date), header.getSchemaName(), header.getTableName() }));
}
}
System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
}
});
buffer.start();
try {
for (int i = 0; i < transactionSize * 10; i++) {
if (i % transactionSize == 0) {
buffer.add(buildEntry("1", 1L + i, 40L + i, EntryType.TRANSACTIONBEGIN));
} else if ((i + 1) % transactionSize == 0) {
buffer.add(buildEntry("1", 1L + i, 40L + i, EntryType.TRANSACTIONEND));
} else {
buffer.add(buildEntry("1", 1L + i, 40L + i));
}
}
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
buffer.stop();
}
@Test
public void testForceFlush() {
final int bufferSize = 64;
EventTransactionBuffer buffer = new EventTransactionBuffer();
buffer.setBufferSize(bufferSize);
buffer.setFlushCallback(new TransactionFlushCallback() {
public void flush(List<Entry> transaction) throws InterruptedException {
Assert.assertEquals(bufferSize, transaction.size());
System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
for (Entry data : transaction) {
CanalEntry.Header header = data.getHeader();
Date date = new Date(header.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] {
// Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae, new Object[] {
Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(),
format.format(date), header.getSchemaName(), header.getTableName() }));
}
}
System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
}
});
buffer.start();
try {
for (int i = 0; i < bufferSize * 2 + 1; i++) {
buffer.add(buildEntry("1", 1L + i, 40L + i));
}
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
buffer.stop();
}
private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
Header.Builder headerBuilder = Header.newBuilder();
headerBuilder.setLogfileName(binlogFile);
headerBuilder.setLogfileOffset(offset);
headerBuilder.setExecuteTime(timestamp);
Entry.Builder entryBuilder = Entry.newBuilder();
entryBuilder.setHeader(headerBuilder.build());
return entryBuilder.build();
}
private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
Header.Builder headerBuilder = Header.newBuilder();
headerBuilder.setLogfileName(binlogFile);
headerBuilder.setLogfileOffset(offset);
headerBuilder.setExecuteTime(timestamp);
Entry.Builder entryBuilder = Entry.newBuilder();
entryBuilder.setHeader(headerBuilder.build());
entryBuilder.setEntryType(type);
return entryBuilder.build();
}
}

33
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java Normal file
View File

@@ -0,0 +1,33 @@
package com.alibaba.otter.canal.parse.inbound;
import java.io.IOException;
import java.net.InetSocketAddress;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
public class TableMetaCacheTest {
@Test
public void testSimple() {
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
try {
connection.connect();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
TableMetaCache cache = new TableMetaCache(connection);
TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
Assert.assertNotNull(meta);
for (FieldMeta field : meta.getFileds()) {
System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : "
+ field.isNullable());
}
}
}

158
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java Normal file
View File

@@ -0,0 +1,158 @@
package com.alibaba.otter.canal.parse.inbound.group;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.store.CanalEventStore;
import com.alibaba.otter.canal.store.CanalStoreException;
import com.alibaba.otter.canal.store.model.Event;
import com.alibaba.otter.canal.store.model.Events;
public class DummyEventStore implements CanalEventStore<Event> {
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static final String messgae = "{0} [{1}:{2}:{3}] {4} {5}.{6}";
public void ack(Position position) throws CanalStoreException {
}
public Events get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
return null;
}
public Events get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
CanalStoreException {
return null;
}
public Position getFirstPosition() throws CanalStoreException {
return null;
}
public Position getLatestPosition() throws CanalStoreException {
return null;
}
public void rollback() throws CanalStoreException {
}
public Events tryGet(Position start, int batchSize) throws CanalStoreException {
return null;
}
public boolean isStart() {
return false;
}
public void start() {
}
public void stop() {
}
public void cleanAll() throws CanalStoreException {
}
public void cleanUntil(Position position) throws CanalStoreException {
}
public void put(Event data) throws InterruptedException, CanalStoreException {
put(Arrays.asList(data));
}
public boolean put(Event data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
return put(Arrays.asList(data), timeout, unit);
}
public boolean tryPut(Event data) throws CanalStoreException {
return tryPut(Arrays.asList(data));
}
public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(),
header.getLogfileName(),
String.valueOf(header.getLogfileOffset()),
format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
}
}
}
public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(),
header.getLogfileName(),
String.valueOf(header.getLogfileOffset()),
format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
}
}
return true;
}
public boolean tryPut(List<Event> datas) throws CanalStoreException {
System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(),
header.getLogfileName(),
String.valueOf(header.getLogfileOffset()),
format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
}
}
System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
return true;
}
}

103
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java Normal file
View File

@@ -0,0 +1,103 @@
package com.alibaba.otter.canal.parse.inbound.group;
import java.net.InetSocketAddress;
import org.junit.Test;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.AbstractBinlogParser;
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
import com.taobao.tddl.dbsync.binlog.LogEvent;
public class GroupEventPaserTest {
private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
private static final String MYSQL_ADDRESS = "127.0.0.1";
private static final String USERNAME = "xxxxx";
private static final String PASSWORD = "xxxxx";
@Test
public void testMysqlWithMysql() {
// MemoryEventStoreWithBuffer eventStore = new
// MemoryEventStoreWithBuffer();
// eventStore.setBufferSize(8196);
GroupEventSink eventSink = new GroupEventSink(3);
eventSink.setFilterTransactionEntry(false);
eventSink.setEventStore(new DummyEventStore());
eventSink.start();
// 构造第一个mysql
MysqlEventParser mysqlEventPaser1 = buildEventParser(3344);
mysqlEventPaser1.setEventSink(eventSink);
// 构造第二个mysql
MysqlEventParser mysqlEventPaser2 = buildEventParser(3345);
mysqlEventPaser2.setEventSink(eventSink);
// 构造第二个mysql
MysqlEventParser mysqlEventPaser3 = buildEventParser(3346);
mysqlEventPaser3.setEventSink(eventSink);
// 启动
mysqlEventPaser1.start();
mysqlEventPaser2.start();
mysqlEventPaser3.start();
try {
Thread.sleep(30 * 10 * 1000L);
} catch (InterruptedException e) {
}
mysqlEventPaser1.stop();
mysqlEventPaser2.stop();
mysqlEventPaser3.stop();
}
private MysqlEventParser buildEventParser(int slaveId) {
MysqlEventParser mysqlEventPaser = new MysqlEventParser();
EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
mysqlEventPaser.setDestination("group-" + slaveId);
mysqlEventPaser.setSlaveId(slaveId);
mysqlEventPaser.setDetectingEnable(false);
mysqlEventPaser.setDetectingSQL(DETECTING_SQL);
mysqlEventPaser.setMasterInfo(buildAuthentication());
mysqlEventPaser.setMasterPosition(defaultPosition);
mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
mysqlEventPaser.setEventSink(new EntryEventSink());
mysqlEventPaser.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
// System.out.println(logPosition);
}
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
return mysqlEventPaser;
}
private BinlogParser buildParser(AuthenticationInfo info) {
return new AbstractBinlogParser<LogEvent>() {
public Entry parse(LogEvent event) throws CanalParseException {
// return _parser.parse(event);
return null;
}
};
}
private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
return new EntryPosition(binlogFile, offest, timestamp);
}
private AuthenticationInfo buildAuthentication() {
return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
}
}

111
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java Normal file
View File

@@ -0,0 +1,111 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.List;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class LocalBinlogDumpTest {
@Test
public void testSimple() {
String directory = "/home/jianghang/tmp/binlog";
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.000006", 4L);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setDirectory(directory);
controller.setMasterPosition(startPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException,
InterruptedException {
for (Entry entry : entrys) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == EntryType.TRANSACTIONEND) {
continue;
}
if (entry.getEntryType() == EntryType.ROWDATA) {
RowChange rowChage = null;
try {
rowChage = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+ entry.toString(), e);
}
EventType eventType = rowChage.getEventType();
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
entry.getHeader().getLogfileName(),
entry.getHeader().getLogfileOffset(),
entry.getHeader().getSchemaName(),
entry.getHeader().getTableName(),
eventType));
for (RowData rowData : rowChage.getRowDatasList()) {
if (eventType == EventType.DELETE) {
print(rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
print(rowData.getAfterColumnsList());
} else {
System.out.println("-------> before");
print(rowData.getBeforeColumnsList());
System.out.println("-------> after");
print(rowData.getAfterColumnsList());
}
}
}
}
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
try {
Thread.sleep(100 * 1000L);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
controller.stop();
}
private void print(List<Column> columns) {
for (Column column : columns) {
System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated());
}
}
}

238
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java Normal file
View File

@@ -0,0 +1,238 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
import java.io.File;
import java.net.InetSocketAddress;
import java.net.URL;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class LocalBinlogEventParserTest {
private static final String MYSQL_ADDRESS = "127.0.0.1";
private static final String USERNAME = "xxxxx";
private static final String PASSWORD = "xxxxx";
private String directory;
@Before
public void setUp() {
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
directory = new File(dummyFile.getParent() + "/binlog").getPath();
}
@Test
public void test_position() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker();
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
controller.setDirectory(directory);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
entryCount.incrementAndGet();
for (Entry entry : entrys) {
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
assertEquals(entryPosition, defaultPosition);
}
@Test
public void test_timestamp() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(300 * 1000);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, 1322803601000L);
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
controller.setDirectory(directory);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
@Override
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
assertTrue(entryPosition.getPosition() <= 6163L);
assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
@Test
public void test_no_position() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000);
final EntryPosition defaultPosition = buildPosition("mysql-bin.000002",
null,
new Date().getTime() + 1000 * 1000L);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
controller.setDirectory(directory);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
// assertEquals(entryPosition.getJournalName(), "mysql-bin.000002");
assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
return new EntryPosition(binlogFile, offest, timestamp);
}
private AuthenticationInfo buildAuthentication() {
return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
}
}

114
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java Normal file
View File

@@ -0,0 +1,114 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.List;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class MysqlDumpTest {
@Test
public void testSimple() {
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L);
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
controller.setMasterPosition(startPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException,
InterruptedException {
for (Entry entry : entrys) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == EntryType.TRANSACTIONEND
|| entry.getEntryType() == EntryType.HEARTBEAT) {
continue;
}
RowChange rowChage = null;
try {
rowChage = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+ entry.toString(), e);
}
EventType eventType = rowChage.getEventType();
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
entry.getHeader().getLogfileName(),
entry.getHeader().getLogfileOffset(),
entry.getHeader().getSchemaName(),
entry.getHeader().getTableName(),
eventType));
if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
System.out.println(" sql ----> " + rowChage.getSql());
}
for (RowData rowData : rowChage.getRowDatasList()) {
if (eventType == EventType.DELETE) {
print(rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
print(rowData.getAfterColumnsList());
} else {
System.out.println("-------> before");
print(rowData.getBeforeColumnsList());
System.out.println("-------> after");
print(rowData.getAfterColumnsList());
}
}
}
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
try {
Thread.sleep(100 * 1000L);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
controller.stop();
}
private void print(List<Column> columns) {
for (Column column : columns) {
System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated());
}
}
}

308
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java Normal file
View File

@@ -0,0 +1,308 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
import java.net.InetSocketAddress;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class MysqlEventParserTest {
private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
private static final String MYSQL_ADDRESS = "127.0.0.1";
private static final String USERNAME = "xxxxx";
private static final String PASSWORD = "xxxxx";
@Test
public void test_position() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker();
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
controller.setSlaveId(3344L);
controller.setDetectingEnable(true);
controller.setDetectingSQL(DETECTING_SQL);
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
@Override
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
Assert.assertEquals(entryPosition, defaultPosition);
}
@Test
public void test_timestamp() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition defaultPosition = buildPosition(null, null, 1322803601000L);
controller.setSlaveId(3344L);
controller.setDetectingEnable(true);
controller.setDetectingSQL(DETECTING_SQL);
controller.setMasterInfo(buildAuthentication());
controller.setMasterPosition(defaultPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
@Override
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
assertTrue(entryPosition.getPosition() <= 6163L);
assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
@Test
public void test_ha() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setMasterInfo(buildAuthentication());
controller.setMasterPosition(defaultPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
@Override
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
public LogPosition getLatestIndexBy(String destination) {
LogPosition masterLogPosition = new LogPosition();
masterLogPosition.setIdentity(new LogIdentity(new InetSocketAddress("127.0.0.1", 3306), 1234L));
masterLogPosition.setPostion(new EntryPosition(1322803601000L));
return masterLogPosition;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
assertTrue(entryPosition.getPosition() <= 6163L);
assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
@Test
public void test_no_position() throws InterruptedException { // 在某个文件下，找不到对应的timestamp数据，会使用106L
// position进行数据抓取
final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 60 * 1000);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001",
null,
new Date().getTime() + 1000 * 1000L);
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setMasterInfo(buildAuthentication());
controller.setMasterPosition(defaultPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
@Override
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (Entry entry : entrys) {
entryCount.incrementAndGet();
// String logfilename = entry.getHeader().getLogfileName();
// long logfileoffset =
// entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
// entryPosition.setJournalName(logfilename);
// entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
controller.stop();
timeoutChecker.stop();
timeoutChecker.touch();
return true;
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
// Assert.assertEquals(logfilename, "mysql-bin.000001");
// Assert.assertEquals(106L, logfileoffset);
assertTrue(entryPosition.getTimestamp() < defaultPosition.getTimestamp());
}
// ======================== helper method =======================
private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
return new EntryPosition(binlogFile, offest, timestamp);
}
private AuthenticationInfo buildAuthentication() {
return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
}
}

170
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java Normal file
View File

@@ -0,0 +1,170 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult;
public class SimpleDdlParserTest {
@Test
public void testCreate() {
String queryString = "CREATE TABLE retl_mark ( `ID` int(11)";
DdlResult result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "CREATE TABLE IF NOT EXISTS retl.retl_mark ( `ID` int(11)";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "CREATE TABLE IF NOT EXISTS `retl_mark` ( `ID` int(11)";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "CREATE TABLE IF NOT EXISTS `retl.retl_mark` ( `ID` int(11)";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "CREATE TABLE `retl`.`retl_mark` (\n `ID` int(10) unsigned NOT NULL";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "CREATE TABLE `retl`.`retl_mark`(\n `ID` int(10) unsigned NOT NULL";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "CREATE table `bak591`.`j_order_log_back_201309` like j_order_log";
result = SimpleDdlParser.parse(queryString, "bak");
Assert.assertNotNull(result);
Assert.assertEquals("bak591", result.getSchemaName());
Assert.assertEquals("j_order_log_back_201309", result.getTableName());
queryString = "CREATE TABLE `bak591`.`cm_settle_incash` ( `batch_id` bigint(20) NOT NULL DEFAULT '0.00'";
result = SimpleDdlParser.parse(queryString, "bak");
Assert.assertNotNull(result);
Assert.assertEquals("bak591", result.getSchemaName());
Assert.assertEquals("cm_settle_incash", result.getTableName());
}
@Test
public void testDrop() {
String queryString = "DROP TABLE retl_mark";
DdlResult result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "DROP TABLE IF EXISTS retl.retl_mark;";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "DROP TABLE IF EXISTS \n `retl.retl_mark` /;";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "DROP /*!40005 TEMPORARY */ /*!40005 TEMPORARY */ TABLE IF EXISTS `temp_bond_keys`.`temp_bond_key_id`;";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("temp_bond_keys", result.getSchemaName());
Assert.assertEquals("temp_bond_key_id", result.getTableName());
}
@Test
public void testAlert() {
String queryString = "alter table retl_mark drop index emp_name";
DdlResult result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "alter table retl.retl_mark drop index emp_name";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "alter table \n `retl.retl_mark` drop index emp_name;";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
}
@Test
public void testTruncate() {
String queryString = "truncate table retl_mark";
DdlResult result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "truncate table retl.retl_mark";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "truncate \n `retl.retl_mark` ";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl_mark", result.getTableName());
}
@Test
public void testRename() {
String queryString = "rename table retl_mark to retl_mark2";
DdlResult result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getOriSchemaName());
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getOriTableName());
Assert.assertEquals("retl_mark2", result.getTableName());
queryString = "rename table retl.retl_mark to retl2.retl_mark2";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getOriSchemaName());
Assert.assertEquals("retl2", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getOriTableName());
Assert.assertEquals("retl_mark2", result.getTableName());
queryString = "rename \n table \n `retl`.`retl_mark` to `retl2.retl_mark2`;";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getOriSchemaName());
Assert.assertEquals("retl2", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getOriTableName());
Assert.assertEquals("retl_mark2", result.getTableName());
}
@Test
public void testIndex() {
String queryString = "CREATE UNIQUE INDEX index_1 ON retl_mark(id,x)";
DdlResult result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
queryString = "create index idx_qca_cid_mcid on q_contract_account (contract_id,main_contract_id)";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("q_contract_account", result.getTableName());
queryString = "DROP INDEX index_str ON retl_mark";
result = SimpleDdlParser.parse(queryString, "retl");
Assert.assertNotNull(result);
Assert.assertEquals("retl", result.getSchemaName());
Assert.assertEquals("retl_mark", result.getTableName());
}
}

38
parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,38 @@
package com.alibaba.otter.canal.parse.index;
import java.net.InetSocketAddress;
import java.util.Date;
import junit.framework.Assert;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
public class AbstractLogPositionManagerTest extends AbstractZkTest {
private static final String MYSQL_ADDRESS = "127.0.0.1";
public LogPosition doTest(CanalLogPositionManager logPositionManager) {
LogPosition getPosition = logPositionManager.getLatestIndexBy(destination);
Assert.assertNull(getPosition);
LogPosition postion1 = buildPosition(1);
logPositionManager.persistLogPosition(destination, postion1);
LogPosition getPosition1 = logPositionManager.getLatestIndexBy(destination);
Assert.assertEquals(postion1, getPosition1);
LogPosition postion2 = buildPosition(2);
logPositionManager.persistLogPosition(destination, postion2);
LogPosition getPosition2 = logPositionManager.getLatestIndexBy(destination);
Assert.assertEquals(postion2, getPosition2);
return postion2;
}
protected LogPosition buildPosition(int number) {
LogPosition position = new LogPosition();
position.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
position.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
return position;
}
}

18
parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java Normal file
View File

@@ -0,0 +1,18 @@
package com.alibaba.otter.canal.parse.index;
import org.junit.Assert;
public class AbstractZkTest {
protected String destination = "ljhtest1";
protected String cluster1 = "127.0.0.1:2188";
protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188";
public void sleep(long time) {
try {
Thread.sleep(time);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
}
}

49
parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,49 @@
package com.alibaba.otter.canal.parse.index;
import java.io.File;
import java.io.IOException;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.protocol.position.LogPosition;
public class FileMixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
private static final String tmp = System.getProperty("java.io.tmpdir", "/tmp");
private static final File dataDir = new File(tmp, "canal");
@Before
public void setUp() {
try {
FileUtils.deleteDirectory(dataDir);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
@Test
public void testAll() {
FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager();
logPositionManager.setDataDir(dataDir);
logPositionManager.setPeriod(100);
logPositionManager.start();
LogPosition position2 = doTest(logPositionManager);
sleep(1500);
FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager();
logPositionManager2.setDataDir(dataDir);
logPositionManager2.setPeriod(100);
logPositionManager2.start();
LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
Assert.assertEquals(position2, getPosition2);
logPositionManager.stop();
logPositionManager2.stop();
}
}

14
parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,14 @@
package com.alibaba.otter.canal.parse.index;
import org.junit.Test;
public class MemoryLogPositionManagerTest extends AbstractLogPositionManagerTest {
@Test
public void testAll() {
MemoryLogPositionManager logPositionManager = new MemoryLogPositionManager();
logPositionManager.start();
doTest(logPositionManager);
logPositionManager.stop();
}
}

88
parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,88 @@
package com.alibaba.otter.canal.parse.index;
import java.net.InetSocketAddress;
import java.util.Date;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.meta.MixedMetaManager;
import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class MetaLogPositionManagerTest extends AbstractLogPositionManagerTest {
private static final String MYSQL_ADDRESS = "127.0.0.1";
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testAll() {
MixedMetaManager metaManager = new MixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
MetaLogPositionManager logPositionManager = new MetaLogPositionManager();
logPositionManager.setMetaManager(metaManager);
logPositionManager.start();
// 构建meta信息
ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
metaManager.subscribe(client1);
PositionRange range1 = buildRange(1);
metaManager.updateCursor(client1, range1.getEnd());
PositionRange range2 = buildRange(2);
metaManager.updateCursor(client1, range2.getEnd());
ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
metaManager.subscribe(client2);
PositionRange range3 = buildRange(3);
metaManager.updateCursor(client2, range3.getEnd());
PositionRange range4 = buildRange(4);
metaManager.updateCursor(client2, range4.getEnd());
LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
Assert.assertEquals(range2.getEnd(), logPosition);
metaManager.stop();
logPositionManager.stop();
}
private PositionRange<LogPosition> buildRange(int number) {
LogPosition start = new LogPosition();
start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
LogPosition end = new LogPosition();
end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
return new PositionRange<LogPosition>(start, end);
}
}

52
parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,52 @@
package com.alibaba.otter.canal.parse.index;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.protocol.position.LogPosition;
public class MixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testAll() {
MixedLogPositionManager logPositionManager = new MixedLogPositionManager();
ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
zookeeperLogPositionManager.setZkClientx(zkclientx);
logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
logPositionManager.start();
LogPosition position2 = doTest(logPositionManager);
sleep(1000);
MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager();
logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
logPositionManager2.start();
LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
Assert.assertEquals(position2, getPosition2);
logPositionManager.stop();
logPositionManager2.stop();
}
}

52
parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,52 @@
package com.alibaba.otter.canal.parse.index;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.protocol.position.LogPosition;
public class PeriodMixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testAll() {
PeriodMixedLogPositionManager logPositionManager = new PeriodMixedLogPositionManager();
ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
zookeeperLogPositionManager.setZkClientx(zkclientx);
logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
logPositionManager.start();
LogPosition position2 = doTest(logPositionManager);
sleep(1500);
PeriodMixedLogPositionManager logPositionManager2 = new PeriodMixedLogPositionManager();
logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
logPositionManager2.start();
LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
Assert.assertEquals(position2, getPosition2);
logPositionManager.stop();
logPositionManager2.stop();
}
}

35
parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java Normal file
View File

@@ -0,0 +1,35 @@
package com.alibaba.otter.canal.parse.index;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
public class ZooKeeperLogPositionManagerTest extends AbstractLogPositionManagerTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testAll() {
ZooKeeperLogPositionManager logPositionManager = new ZooKeeperLogPositionManager();
logPositionManager.setZkClientx(zkclientx);
logPositionManager.start();
doTest(logPositionManager);
logPositionManager.stop();
}
}

11
parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java Normal file
View File

@@ -0,0 +1,11 @@
package com.alibaba.otter.canal.parse.stub;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.sink.CanalEventSink;
public abstract class AbstractCanalEventSinkTest<T> extends AbstractCanalLifeCycle implements CanalEventSink<T> {
public void interrupt() {
// do nothing
}
}

8
parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java Normal file
View File

@@ -0,0 +1,8 @@
package com.alibaba.otter.canal.parse.stub;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
public abstract class AbstractCanalLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
}

BIN
parse/src/test/resources/binlog/mysql-bin.000001 Normal file
View File

Binary file not shown.

BIN
parse/src/test/resources/binlog/mysql-bin.000002 Normal file
View File

Binary file not shown.

1
parse/src/test/resources/dummy.txt Normal file
View File

@@ -0,0 +1 @@
本文件仅仅为定位绝对路径使用