chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rds binlog 下载支持

Browse Source
This commit is contained in:
charles.lin
2018-08-08 19:29:52 +08:00
parent c6eee47af6
commit 61c7f7738d
131 changed files with 7250 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java Normal file
View File

@@ -0,0 +1,89 @@
package com.alibaba.otter.canal.parse;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class MysqlBinlogDumpPerformanceTest {
public static void main(String args[]) {
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.001699", 120L, 100L);
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setFilterQueryDml(true);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3328), "root", "hello"));
controller.setMasterPosition(startPosition);
controller.setEnableTsdb(false);
controller.setDestination("example");
controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
// controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
// controller.setEventBlackFilter(new
// AviaterRegexFilter("canal_tsdb\\..*"));
controller.setParallel(true);
controller.setParallelBufferSize(256);
controller.setParallelThreadSize(16);
controller.setIsGTIDMode(false);
final AtomicLong sum = new AtomicLong(0);
final AtomicLong last = new AtomicLong(0);
final AtomicLong start = new AtomicLong(System.currentTimeMillis());
final AtomicLong end = new AtomicLong(0);
controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException,
InterruptedException {
sum.addAndGet(entrys.size());
long current = sum.get();
if (current - last.get() >= 100000) {
end.set(System.currentTimeMillis());
long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
last.set(current);
start.set(end.get());
}
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
}
});
controller.start();
try {
Thread.sleep(100 * 1000 * 1000L);
} catch (InterruptedException e) {
}
controller.stop();
}
public static abstract class AbstractCanalEventSinkTest<T> extends AbstractCanalLifeCycle implements CanalEventSink<T> {
public void interrupt() {
}
}
}

82
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java Normal file
View File

@@ -0,0 +1,82 @@
package com.alibaba.otter.canal.parse;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
public class MysqlBinlogEventPerformanceTest {
protected static Charset charset = Charset.forName("utf-8");
public static void main(String args[]) {
DirectLogFetcher fetcher = new DirectLogFetcher();
try {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
connector.connect();
updateSettings(connector);
sendBinlogDump(connector, "mysql-bin.000006", 120L, 3);
fetcher.start(connector.getChannel());
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
AtomicLong sum = new AtomicLong(0);
long start = System.currentTimeMillis();
long last = 0;
long end = 0;
while (fetcher.fetch()) {
decoder.decode(fetcher, context);
sum.incrementAndGet();
long current = sum.get();
if (current - last >= 100000) {
end = System.currentTimeMillis();
long tps = ((current - last) * 1000) / (end - start);
System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
last = current;
start = end;
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
fetcher.close();
} catch (IOException e) {
}
}
}
private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
throws IOException {
BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
binlogDumpCmd.binlogFileName = binlogfilename;
binlogDumpCmd.binlogPosition = binlogPosition;
binlogDumpCmd.slaveServerId = slaveId;
byte[] cmdBody = binlogDumpCmd.toBytes();
HeaderPacket binlogDumpHeader = new HeaderPacket();
binlogDumpHeader.setPacketBodyLength(cmdBody.length);
binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody);
}
private static void updateSettings(MysqlConnector connector) throws IOException {
update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector);
}
public static void update(String cmd, MysqlConnector connector) throws IOException {
MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
exector.update(cmd);
}
}

192
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java Normal file
View File

@@ -0,0 +1,192 @@
package com.alibaba.otter.canal.parse;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.BitSet;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
public class MysqlBinlogParsePerformanceTest {
protected static Charset charset = Charset.forName("utf-8");
public static void main(String args[]) {
DirectLogFetcher fetcher = new DirectLogFetcher();
try {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
connector.connect();
updateSettings(connector);
sendBinlogDump(connector, "mysql-bin.000006", 120L, 3);
fetcher.start(connector.getChannel());
final BlockingQueue<LogBuffer> buffer = new ArrayBlockingQueue<LogBuffer>(1024);
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
consumer(buffer);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
thread.start();
while (fetcher.fetch()) {
buffer.put(fetcher.duplicate());
fetcher.consume(fetcher.limit());
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
fetcher.close();
} catch (IOException e) {
}
}
}
public static void consumer(BlockingQueue<LogBuffer> buffer) throws IOException, InterruptedException {
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
AtomicLong sum = new AtomicLong(0);
long start = System.currentTimeMillis();
long last = 0;
long end = 0;
while (true) {
LogEvent event = null;
event = decoder.decode(buffer.take(), context);
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
parseRowsEvent((WriteRowsLogEvent) event, sum);
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
parseRowsEvent((UpdateRowsLogEvent) event, sum);
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
parseRowsEvent((DeleteRowsLogEvent) event, sum);
break;
case LogEvent.XID_EVENT:
sum.incrementAndGet();
break;
case LogEvent.QUERY_EVENT:
sum.incrementAndGet();
break;
default:
break;
}
long current = sum.get();
if (current - last >= 100000) {
end = System.currentTimeMillis();
long tps = ((current - last) * 1000) / (end - start);
System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
last = current;
start = end;
}
}
}
private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
throws IOException {
BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
binlogDumpCmd.binlogFileName = binlogfilename;
binlogDumpCmd.binlogPosition = binlogPosition;
binlogDumpCmd.slaveServerId = slaveId;
byte[] cmdBody = binlogDumpCmd.toBytes();
HeaderPacket binlogDumpHeader = new HeaderPacket();
binlogDumpHeader.setPacketBodyLength(cmdBody.length);
binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody);
}
private static void updateSettings(MysqlConnector connector) throws IOException {
update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector);
update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector);
}
public static void update(String cmd, MysqlConnector connector) throws IOException {
MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
exector.update(cmd);
}
public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
try {
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
while (buffer.nextOneRow(columns)) {
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
parseOneRow(event, buffer, columns, true);
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
parseOneRow(event, buffer, columns, false);
} else {
parseOneRow(event, buffer, columns, false);
if (!buffer.nextOneRow(changeColumns)) {
break;
}
parseOneRow(event, buffer, changeColumns, true);
}
sum.incrementAndGet();
}
} catch (Exception e) {
throw new RuntimeException("parse row data failed.", e);
}
}
public static void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
throws UnsupportedEncodingException {
TableMapLogEvent map = event.getTable();
if (map == null) {
throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
}
final int columnCnt = map.getColumnCnt();
final ColumnInfo[] columnInfo = map.getColumnInfo();
for (int i = 0; i < columnCnt; i++) {
if (!cols.get(i)) {
continue;
}
ColumnInfo info = columnInfo[i];
buffer.nextValue(info.type, info.meta);
if (buffer.isNull()) {
} else {
buffer.getValue();
}
}
}
}

3
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
View File

@@ -41,6 +41,9 @@ public class MysqlDumpTest {
controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml");
controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*"));
controller.setParallel(true);
controller.setParallelBufferSize(256);
controller.setParallelThreadSize(2);
controller.setIsGTIDMode(false);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

153
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java Normal file
View File

@@ -0,0 +1,153 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
/**
* @author chengjin.lyf on 2018/7/21 下午5:24
* @since 1.0.25
*/
public class RdsBinlogEventParserProxyTest {
private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
private static final String MYSQL_ADDRESS = "";
private static final String USERNAME = "";
private static final String PASSWORD = "";
public static final String DBNAME = "";
public static final String TBNAME = "";
public static final String DDL = "";
@Test
public void test_timestamp() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, -1);
final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setDetectingSQL(DETECTING_SQL);
controller.setMasterInfo(buildAuthentication());
controller.setMasterPosition(defaultPosition);
controller.setInstanceId("");
controller.setAccesskey("");
controller.setSecretkey("");
controller.setBatchSize(4);
// controller.setRdsOpenApiUrl("https://rds.aliyuncs.com/");
controller.setTableMetaStorageFactory(new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
@Override
public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
}
@Override
public List<TableMetaEntry> fetch(String dbAddress, String dbName) {
TableMetaEntry tableMeta = new TableMetaEntry();
tableMeta.setSchema(DBNAME);
tableMeta.setTable(TBNAME);
tableMeta.setDdl(DDL);
tableMeta.setTimestamp(new Date().getTime());
List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
entries.add(tableMeta);
return entries;
}
@Override
public List<TableMetaEntry> fetch(String dbAddress, String dbName, String tableName) {
TableMetaEntry tableMeta = new TableMetaEntry();
tableMeta.setSchema(DBNAME);
tableMeta.setTable(TBNAME);
tableMeta.setDdl(DDL);
tableMeta.setTimestamp(new Date().getTime());
List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
entries.add(tableMeta);
return entries;
}
}, DBNAME));
controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
@Override
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
for (CanalEntry.Entry entry : entrys) {
if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
entryCount.incrementAndGet();
String logfilename = entry.getHeader().getLogfileName();
long logfileoffset = entry.getHeader().getLogfileOffset();
long executeTime = entry.getHeader().getExecuteTime();
entryPosition.setJournalName(logfilename);
entryPosition.setPosition(logfileoffset);
entryPosition.setTimestamp(executeTime);
break;
}
}
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
private LogPosition logPosition;
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
this.logPosition = logPosition;
}
public LogPosition getLatestIndexBy(String destination) {
return logPosition;
}
});
controller.start();
timeoutChecker.waitForIdle();
if (controller.isStart()) {
controller.stop();
}
// check
Assert.assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
Assert.assertTrue(entryPosition.getPosition() <= 6163L);
Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
// ======================== helper method =======================
private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
return new EntryPosition(binlogFile, offest, timestamp);
}
private AuthenticationInfo buildAuthentication() {
return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
}
}

38
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java Normal file
View File

@@ -0,0 +1,38 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import org.junit.Test;
import java.net.InetSocketAddress;
import java.util.Date;
public class NoStorageTest {
final String DBNAME = "testdb";
final String TBNAME = "testtb";
final String DDL = "CREATE TABLE `testtb` (\n" +
" `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
" `name` varchar(2048) DEFAULT NULL,\n" +
" `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
" `otter_testcol` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
" PRIMARY KEY (`id`)\n" +
" ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
@Test
public void nostorage() {
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, null);
EntryPosition entryPosition = new EntryPosition();
entryPosition.setTimestamp(new Date().getTime());
String fullTableName = DBNAME + "." + TBNAME;
tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
entryPosition.setTimestamp(new Date().getTime() + 1000L);
TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
assert result.getDdl().equalsIgnoreCase(DDL);
}
}

77
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java Normal file
View File

@@ -0,0 +1,77 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import org.junit.Test;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class StorageTest {
final String DBNAME = "testdb";
final String TBNAME = "testtb";
final String DDL = "CREATE TABLE `testtb` (\n" +
" `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
" `name` varchar(2048) DEFAULT NULL,\n" +
" `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
" `otter_testcol` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
" PRIMARY KEY (`id`)\n" +
" ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
@Test
public void storage() {
MySqlTableMetaStorageFactory factory = new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
@Override
public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
}
@Override
public List<TableMetaEntry> fetch(String dbAddress, String dbName) {
TableMetaEntry tableMeta = new TableMetaEntry();
tableMeta.setSchema(DBNAME);
tableMeta.setTable(TBNAME);
tableMeta.setDdl(DDL);
tableMeta.setTimestamp(new Date().getTime());
List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
entries.add(tableMeta);
return entries;
}
@Override
public List<TableMetaEntry> fetch(String dbAddress, String dbName, String tableName) {
TableMetaEntry tableMeta = new TableMetaEntry();
tableMeta.setSchema(DBNAME);
tableMeta.setTable(TBNAME);
tableMeta.setDdl(DDL);
tableMeta.setTimestamp(new Date().getTime());
List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
entries.add(tableMeta);
return entries;
}
}, DBNAME);
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, factory.getTableMetaStorage());
EntryPosition entryPosition = new EntryPosition();
entryPosition.setTimestamp(new Date().getTime());
String fullTableName = DBNAME + "." + TBNAME;
tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
entryPosition.setTimestamp(new Date().getTime() + 1000L);
TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
assert result.getDdl().equalsIgnoreCase(DDL);
}
}