chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java Normal file
View File

@@ -0,0 +1,109 @@
package com.taobao.tddl.dbsync.binlog;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.BitSet;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
public class BaseLogFetcherTest {
protected String binlogFileName = "mysql-bin.000001";
protected Charset charset = Charset.forName("utf-8");
protected void parseQueryEvent(QueryLogEvent event) {
System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getCatalog()));
System.out.println("sql : " + event.getQuery());
}
protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
}
protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
}
protected void parseXidEvent(XidLogEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println("xid : " + event.getXid());
}
protected void parseRowsEvent(RowsLogEvent event) {
try {
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getTable().getDbName(), event.getTable().getTableName()));
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
while (buffer.nextOneRow(columns)) {
// 处理row记录
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
// insert的记录放在before字段中
parseOneRow(event, buffer, columns, true);
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
// delete的记录放在before字段中
parseOneRow(event, buffer, columns, false);
} else {
// update需要处理before/after
System.out.println("-------> before");
parseOneRow(event, buffer, columns, false);
if (!buffer.nextOneRow(changeColumns)) {
break;
}
System.out.println("-------> after");
parseOneRow(event, buffer, changeColumns, true);
}
}
} catch (Exception e) {
throw new RuntimeException("parse row data failed.", e);
}
}
protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
throws UnsupportedEncodingException {
TableMapLogEvent map = event.getTable();
if (map == null) {
throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
}
final int columnCnt = map.getColumnCnt();
final ColumnInfo[] columnInfo = map.getColumnInfo();
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = columnInfo[i];
buffer.nextValue(info.type, info.meta);
if (buffer.isNull()) {
//
} else {
final Serializable value = buffer.getValue();
if (value instanceof byte[]) {
System.out.println(new String((byte[]) value));
} else {
System.out.println(value);
}
}
}
}
}

90
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java Normal file
View File

@@ -0,0 +1,90 @@
package com.taobao.tddl.dbsync.binlog;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import junit.framework.Assert;
import org.junit.Test;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
public class DirectLogFetcherTest extends BaseLogFetcherTest {
@Test
public void testSimple() {
DirectLogFetcher fecther = new DirectLogFetcher();
try {
Class.forName("com.mysql.jdbc.Driver");
Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "xxxxx", "xxxxx");
Statement statement = connection.createStatement();
statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
fecther.open(connection, "mysql-bin.000003", 4L, 2);
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
while (fecther.fetch()) {
LogEvent event = null;
event = decoder.decode(fecther, context);
if (event == null) {
throw new RuntimeException("parse failed");
}
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
binlogFileName = ((RotateLogEvent) event).getFilename();
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
parseRowsEvent((WriteRowsLogEvent) event);
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
parseRowsEvent((UpdateRowsLogEvent) event);
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
parseRowsEvent((DeleteRowsLogEvent) event);
break;
case LogEvent.QUERY_EVENT:
parseQueryEvent((QueryLogEvent) event);
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
parseRowsQueryEvent((RowsQueryLogEvent) event);
break;
case LogEvent.ANNOTATE_ROWS_EVENT:
parseAnnotateRowsEvent((AnnotateRowsEvent) event);
break;
case LogEvent.XID_EVENT:
parseXidEvent((XidLogEvent) event);
break;
default:
break;
}
}
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
} finally {
try {
fecther.close();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
}

93
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java Normal file
View File

@@ -0,0 +1,93 @@
package com.taobao.tddl.dbsync.binlog;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import junit.framework.Assert;
import org.junit.Before;
import org.junit.Test;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
public class FileLogFetcherTest extends BaseLogFetcherTest {
private String directory;
@Before
public void setUp() {
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
directory = new File(dummyFile.getParent() + "/binlog").getPath();
// directory = "/home/jianghang/tmp/binlog";
}
@Test
public void testSimple() {
FileLogFetcher fetcher = new FileLogFetcher(1024 * 16);
try {
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
File current = new File(directory, "mysql-bin.000006");
fetcher.open(current);
context.setLogPosition(new LogPosition(current.getName()));
while (fetcher.fetch()) {
LogEvent event = null;
event = decoder.decode(fetcher, context);
if (event != null) {
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
binlogFileName = ((RotateLogEvent)
event).getFilename();
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
parseRowsEvent((WriteRowsLogEvent) event);
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
parseRowsEvent((UpdateRowsLogEvent) event);
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
parseRowsEvent((DeleteRowsLogEvent) event);
break;
case LogEvent.QUERY_EVENT:
parseQueryEvent((QueryLogEvent) event);
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
parseRowsQueryEvent((RowsQueryLogEvent) event);
break;
case LogEvent.ANNOTATE_ROWS_EVENT:
parseAnnotateRowsEvent((AnnotateRowsEvent) event);
break;
case LogEvent.XID_EVENT:
parseXidEvent((XidLogEvent) event);
break;
default:
break;
}
}
}
} catch (Exception e) {
Assert.fail(e.getMessage());
} finally {
try {
fetcher.close();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
}

356
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java Normal file
View File

@@ -0,0 +1,356 @@
package com.taobao.tddl.dbsync.binlog;
import java.math.BigDecimal;
import java.math.BigInteger;
import junit.framework.TestCase;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
public class LogBufferTest extends TestCase
{
public static final int LOOP = 10000;
public void testSigned()
{
byte[] array = { 0, 0, 0, (byte) 0xff };
LogBuffer buffer = new LogBuffer(array, 0, array.length);
System.out.println(buffer.getInt32(0));
System.out.println(buffer.getUint32(0));
System.out.println(buffer.getInt24(1));
System.out.println(buffer.getUint24(1));
}
public void testBigInteger()
{
byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
LogBuffer buffer = new LogBuffer(array, 0, array.length);
long tt1 = 0;
long l1 = 0;
for (int i = 0; i < LOOP; i++)
{
final long t1 = System.nanoTime();
l1 = buffer.getLong64(0);
tt1 += System.nanoTime() - t1;
}
System.out.print(tt1 / LOOP);
System.out.print("ns >> ");
System.out.println(l1);
long tt2 = 0;
BigInteger l2 = null;
for (int i = 0; i < LOOP; i++)
{
final long t2 = System.nanoTime();
l2 = buffer.getUlong64(0);
tt2 += System.nanoTime() - t2;
}
System.out.print(tt2 / LOOP);
System.out.print("ns >> ");
System.out.println(l2);
}
/* Reads big-endian integer from no more than 4 bytes */
private static int convertNBytesToInt(byte[] buffer, int offset, int length)
{
int ret = 0;
for (int i = offset; i < (offset + length); i++)
{
ret = (ret << 8) | (0xff & buffer[i]);
}
return ret;
}
private static int convert4BytesToInt(byte[] buffer, int offset)
{
int value;
value = (0xff & buffer[offset + 3]);
value += (0xff & buffer[offset + 2]) << 8;
value += (0xff & buffer[offset + 1]) << 16;
value += (0xff & buffer[offset]) << 24;
return value;
}
public static short convert1ByteToShort(byte[] buffer, int offset)
{
short value;
value = (short) buffer[offset + 0];
return value;
}
public static short convert2bytesToShort(byte[] buffer, int offset)
{
short value;
value = (short) (buffer[offset + 0] << 8);
value += (short) (buffer[offset + 1] & 0xff);
return value;
}
public static final BigDecimal extractDecimal(byte[] buffer, int precision,
int scale)
{
//
// Decimal representation in binlog seems to be as follows:
// 1 byte - 'precision'
// 1 byte - 'scale'
// remaining n bytes - integer such that value = n / (10^scale)
// Integer is represented as follows:
// 1st bit - sign such that set == +, unset == -
// every 4 bytes represent 9 digits in big-endian order, so that if
// you print the values of these quads as big-endian integers one after
// another, you get the whole number string representation in decimal.
// What remains is to put a sign and a decimal dot.
// 13 0a 80 00 00 05 1b 38 b0 60 00 means:
// 0x13 - precision = 19
// 0x0a - scale = 10
// 0x80 - positive
// 0x00000005 0x1b38b060 0x00
// 5 456700000 0
// 54567000000 / 10^{10} = 5.4567
//
// int_size below shows how long is integer part
//
// offset = offset + 2; // offset of the number part
//
int intg = precision - scale;
int intg0 = intg / LogBuffer.DIG_PER_INT32;
int frac0 = scale / LogBuffer.DIG_PER_INT32;
int intg0x = intg - intg0 * LogBuffer.DIG_PER_INT32;
int frac0x = scale - frac0 * LogBuffer.DIG_PER_INT32;
int offset = 0;
int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
// how many bytes are used to represent given amount of digits?
int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
+ LogBuffer.dig2bytes[intg0x];
int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
+ LogBuffer.dig2bytes[frac0x];
int bin_size = integerSize + decimalSize; // total bytes
byte[] d_copy = new byte[bin_size];
if (bin_size > buffer.length)
{
throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
+ bin_size + ", available bytes: " + buffer.length);
}
// Invert first bit
d_copy[0] = buffer[0];
d_copy[0] ^= 0x80;
if (sign == -1)
{
// Invert every byte
d_copy[0] ^= 0xFF;
}
for (int i = 1; i < bin_size; i++)
{
d_copy[i] = buffer[i];
if (sign == -1)
{
// Invert every byte
d_copy[i] ^= 0xFF;
}
}
// Integer part
offset = LogBuffer.dig2bytes[intg0x];
BigDecimal intPart = new BigDecimal(0);
if (offset > 0)
intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
while (offset < integerSize)
{
intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
offset += 4;
}
// Decimal part
BigDecimal fracPart = new BigDecimal(0);
int shift = 0;
for (int i = 0; i < frac0; i++)
{
shift += LogBuffer.DIG_PER_DEC1;
fracPart = fracPart.add(BigDecimal.valueOf(
convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
offset += 4;
}
if (LogBuffer.dig2bytes[frac0x] > 0)
{
fracPart = fracPart.add(BigDecimal.valueOf(
convertNBytesToInt(d_copy, offset,
LogBuffer.dig2bytes[frac0x])).movePointLeft(
shift + frac0x));
}
return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
}
public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
0x38, (byte) 0xb0, 0x60, 0x00 };
public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
(byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
(byte) 0xa0, (byte) 0xff };
public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
-108 };
public void testBigDecimal() throws InterruptedException
{
do
{
System.out.println("old extract decimal: ");
long tt1 = 0;
BigDecimal bd1 = null;
for (int i = 0; i < LOOP; i++)
{
final long t1 = System.nanoTime();
bd1 = extractDecimal(array2, 19, 10);
tt1 += System.nanoTime() - t1;
}
System.out.print(tt1 / LOOP);
System.out.print("ns >> ");
System.out.println(bd1);
long tt2 = 0;
BigDecimal bd2 = null;
for (int i = 0; i < LOOP; i++)
{
final long t2 = System.nanoTime();
bd2 = extractDecimal(array1, 19, 10);
tt2 += System.nanoTime() - t2;
}
System.out.print(tt2 / LOOP);
System.out.print("ns >> ");
System.out.println(bd2);
long tt3 = 0;
BigDecimal bd3 = null;
for (int i = 0; i < LOOP; i++)
{
final long t3 = System.nanoTime();
bd3 = extractDecimal(array3, 18, 6);
tt3 += System.nanoTime() - t3;
}
System.out.print(tt3 / LOOP);
System.out.print("ns >> ");
System.out.println(bd3);
long tt4 = 0;
BigDecimal bd4 = null;
for (int i = 0; i < LOOP; i++)
{
final long t4 = System.nanoTime();
bd4 = extractDecimal(array4, 18, 6);
tt4 += System.nanoTime() - t4;
}
System.out.print(tt4 / LOOP);
System.out.print("ns >> ");
System.out.println(bd4);
long tt5 = 0;
BigDecimal bd5 = null;
for (int i = 0; i < LOOP; i++)
{
final long t5 = System.nanoTime();
bd5 = extractDecimal(array5, 18, 6);
tt5 += System.nanoTime() - t5;
}
System.out.print(tt5 / LOOP);
System.out.print("ns >> ");
System.out.println(bd5);
}
while (false);
do
{
System.out.println("new extract decimal: ");
LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
LogBuffer buffer2 = new LogBuffer(array1, 0, array1.length);
LogBuffer buffer3 = new LogBuffer(array3, 0, array3.length);
LogBuffer buffer4 = new LogBuffer(array4, 0, array4.length);
LogBuffer buffer5 = new LogBuffer(array5, 0, array5.length);
long tt1 = 0;
BigDecimal bd1 = null;
for (int i = 0; i < LOOP; i++)
{
final long t1 = System.nanoTime();
bd1 = buffer1.getDecimal(0, 19, 10);
tt1 += System.nanoTime() - t1;
}
System.out.print(tt1 / LOOP);
System.out.print("ns >> ");
System.out.println(bd1);
long tt2 = 0;
BigDecimal bd2 = null;
for (int i = 0; i < LOOP; i++)
{
final long t2 = System.nanoTime();
bd2 = buffer2.getDecimal(0, 19, 10);
tt2 += System.nanoTime() - t2;
}
System.out.print(tt2 / LOOP);
System.out.print("ns >> ");
System.out.println(bd2);
long tt3 = 0;
BigDecimal bd3 = null;
for (int i = 0; i < LOOP; i++)
{
final long t3 = System.nanoTime();
bd3 = buffer3.getDecimal(0, 18, 6);
tt3 += System.nanoTime() - t3;
}
System.out.print(tt3 / LOOP);
System.out.print("ns >> ");
System.out.println(bd3);
long tt4 = 0;
BigDecimal bd4 = null;
for (int i = 0; i < LOOP; i++)
{
final long t4 = System.nanoTime();
bd4 = buffer4.getDecimal(0, 18, 6);
tt4 += System.nanoTime() - t4;
}
System.out.print(tt4 / LOOP);
System.out.print("ns >> ");
System.out.println(bd4);
long tt5 = 0;
BigDecimal bd5 = null;
for (int i = 0; i < LOOP; i++)
{
final long t5 = System.nanoTime();
bd5 = buffer5.getDecimal(0, 18, 6);
tt5 += System.nanoTime() - t5;
}
System.out.print(tt5 / LOOP);
System.out.print("ns >> ");
System.out.println(bd5);
}
while (false);
}
}

BIN
dbsync/src/test/resources/binlog/mysql-bin.000001 Normal file
View File

Binary file not shown.

1
dbsync/src/test/resources/dummy.txt Normal file
View File

@@ -0,0 +1 @@
本文件仅仅为定位绝对路径使用