chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed code template and format

Browse Source
This commit is contained in:
agapple
2015-04-28 21:11:41 +08:00
parent ea889cb004
commit 7ebb3fd1ab
86 changed files with 2763 additions and 3554 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
View File

@@ -6,24 +6,24 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* An utility class implements MySQL/Java charsets conversion. you can see com.mysql.jdbc.CharsetMapping.
* An utility class implements MySQL/Java charsets conversion. you can see
* com.mysql.jdbc.CharsetMapping.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
*/
public final class CharsetConversion
{
public final class CharsetConversion {
static final Log logger = LogFactory.getLog(CharsetConversion.class);
static final class Entry
{
static final class Entry {
protected final int charsetId;
protected final String mysqlCharset;
protected final String mysqlCollation;
protected final String javaCharset;
Entry(final int id, String mysqlCharset, // NL
String mysqlCollation, String javaCharset)
{
String mysqlCollation, String javaCharset){
this.charsetId = id;
this.mysqlCharset = mysqlCharset;
this.mysqlCollation = mysqlCollation;
@@ -34,38 +34,31 @@ public final class CharsetConversion
// Character set data used in lookups. The array will be sparse.
static final Entry[] entries = new Entry[0xff];
static Entry getEntry(final int id)
{
if (id >= 0 && id < entries.length)
{
static Entry getEntry(final int id) {
if (id >= 0 && id < entries.length) {
return entries[id];
}
else
{
} else {
throw new IllegalArgumentException("Invalid charset id: " + id);
}
}
// Loads character set information.
static void putEntry(final int charsetId, String mysqlCharset,
String mysqlCollation, String javaCharset)
{
static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation, String javaCharset) {
entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
mysqlCollation, javaCharset);
mysqlCollation,
javaCharset);
}
// Loads character set information.
@Deprecated
static void putEntry(final int charsetId, String mysqlCharset,
String mysqlCollation)
{
static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation) {
entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
mysqlCollation, /* Unknown java charset */null);
mysqlCollation, /* Unknown java charset */
null);
}
// Load character set data statically.
static
{
static {
putEntry(1, "big5", "big5_chinese_ci", "Big5");
putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
@@ -135,7 +128,7 @@ public final class CharsetConversion
putEntry(69, "dec8", "dec8_bin", "US-ASCII");
putEntry(70, "greek", "greek_bin", "ISO8859_7");
putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
putEntry(72, "hp8", "hp8_bin" , "US-ASCII");
putEntry(72, "hp8", "hp8_bin", "US-ASCII");
putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
@@ -274,16 +267,12 @@ public final class CharsetConversion
/**
* Return defined charset name for mysql.
*/
public static String getCharset(final int id)
{
public static String getCharset(final int id) {
Entry entry = getEntry(id);
if (entry != null)
{
if (entry != null) {
return entry.mysqlCharset;
}
else
{
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
@@ -292,16 +281,12 @@ public final class CharsetConversion
/**
* Return defined collaction name for mysql.
*/
public static String getCollation(final int id)
{
public static String getCollation(final int id) {
Entry entry = getEntry(id);
if (entry != null)
{
if (entry != null) {
return entry.mysqlCollation;
}
else
{
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
@@ -310,55 +295,41 @@ public final class CharsetConversion
/**
* Return converted charset name for java.
*/
public static String getJavaCharset(final int id)
{
public static String getJavaCharset(final int id) {
Entry entry = getEntry(id);
if (entry != null)
{
if (entry.javaCharset != null)
{
if (entry != null) {
if (entry.javaCharset != null) {
return entry.javaCharset;
}
else
{
logger.warn("Unknown java charset for: id = " + id
+ ", name = " + entry.mysqlCharset + ", coll = "
+ entry.mysqlCollation);
} else {
logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
+ entry.mysqlCollation);
return null;
}
}
else
{
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
}
public static void main(String[] args)
{
for (int i = 0; i < entries.length; i++)
{
public static void main(String[] args) {
for (int i = 0; i < entries.length; i++) {
Entry entry = entries[i];
System.out.print(i);
System.out.print(',');
System.out.print(' ');
if (entry != null)
{
if (entry != null) {
System.out.print(entry.mysqlCharset);
System.out.print(',');
System.out.print(' ');
System.out.print(entry.javaCharset);
if (entry.javaCharset != null)
{
if (entry.javaCharset != null) {
System.out.print(',');
System.out.print(' ');
System.out.print(Charset.forName(entry.javaCharset).name());
}
}
else
{
} else {
System.out.print("null");
}
System.out.println();

307
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java
View File

@@ -21,16 +21,13 @@ import org.apache.commons.logging.LogFactory;
* DirectLogFetcher fetcher = new DirectLogFetcher();
* fetcher.open(conn, file, 0, 13);
*
* while (fetcher.fetch())
* {
* while (fetcher.fetch()) {
* LogEvent event;
* do
* {
* do {
* event = decoder.decode(fetcher, context);
*
* // process log event.
* }
* while (event != null);
* } while (event != null);
* }
* // connection closed.
* </pre>
@@ -38,97 +35,79 @@ import org.apache.commons.logging.LogFactory;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class DirectLogFetcher extends LogFetcher
{
protected static final Log logger = LogFactory.getLog(DirectLogFetcher.class);
public final class DirectLogFetcher extends LogFetcher {
protected static final Log logger = LogFactory.getLog(DirectLogFetcher.class);
/** Command to dump binlog */
public static final byte COM_BINLOG_DUMP = 18;
public static final byte COM_BINLOG_DUMP = 18;
/** Packet header sizes */
public static final int NET_HEADER_SIZE = 4;
public static final int SQLSTATE_LENGTH = 5;
public static final int NET_HEADER_SIZE = 4;
public static final int SQLSTATE_LENGTH = 5;
/** Packet offsets */
public static final int PACKET_LEN_OFFSET = 0;
public static final int PACKET_SEQ_OFFSET = 3;
public static final int PACKET_LEN_OFFSET = 0;
public static final int PACKET_SEQ_OFFSET = 3;
/** Maximum packet length */
public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
/** BINLOG_DUMP options */
public static final int BINLOG_DUMP_NON_BLOCK = 1;
public static final int BINLOG_DUMP_NON_BLOCK = 1;
public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
private Connection conn;
private OutputStream mysqlOutput;
private InputStream mysqlInput;
public DirectLogFetcher()
{
public DirectLogFetcher(){
super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
}
public DirectLogFetcher(final int initialCapacity)
{
public DirectLogFetcher(final int initialCapacity){
super(initialCapacity, DEFAULT_GROWTH_FACTOR);
}
public DirectLogFetcher(final int initialCapacity, final float growthFactor)
{
public DirectLogFetcher(final int initialCapacity, final float growthFactor){
super(initialCapacity, growthFactor);
}
private static final Object unwrapConnection(Object conn, Class<?> connClazz)
throws IOException
{
while (!connClazz.isInstance(conn))
{
try
{
private static final Object unwrapConnection(Object conn, Class<?> connClazz) throws IOException {
while (!connClazz.isInstance(conn)) {
try {
Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
if (connProxy.isInstance(conn))
{
if (connProxy.isInstance(conn)) {
conn = invokeMethod(conn, connProxy, "getTargetConnection");
continue;
}
}
catch (ClassNotFoundException e)
{
// org.springframework.jdbc.datasource.ConnectionProxy not found.
} catch (ClassNotFoundException e) {
// org.springframework.jdbc.datasource.ConnectionProxy not
// found.
}
try
{
try {
Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
if (connProxy.isInstance(conn))
{
if (connProxy.isInstance(conn)) {
conn = getDeclaredField(conn, connProxy, "_conn");
continue;
}
}
catch (ClassNotFoundException e)
{
} catch (ClassNotFoundException e) {
// org.apache.commons.dbcp.DelegatingConnection not found.
}
try
{
if (conn instanceof java.sql.Wrapper)
{
try {
if (conn instanceof java.sql.Wrapper) {
Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
conn = ((java.sql.Wrapper) conn).unwrap(connIface);
continue;
}
}
catch (ClassNotFoundException e)
{
} catch (ClassNotFoundException e) {
// com.mysql.jdbc.Connection not found.
}
catch (SQLException e)
{
logger.warn("Unwrap " + conn.getClass().getName() + " to "
+ connClazz.getName() + " failed: " + e.getMessage(), e);
} catch (SQLException e) {
logger.warn("Unwrap " + conn.getClass().getName() + " to " + connClazz.getName() + " failed: "
+ e.getMessage(),
e);
}
return null;
@@ -136,127 +115,85 @@ public final class DirectLogFetcher extends LogFetcher
return conn;
}
private static final Object invokeMethod(Object obj, Class<?> objClazz,
String name)
{
try
{
private static final Object invokeMethod(Object obj, Class<?> objClazz, String name) {
try {
Method method = objClazz.getMethod(name, (Class<?>[]) null);
return method.invoke(obj, (Object[]) null);
}
catch (NoSuchMethodException e)
{
throw new IllegalArgumentException("No such method: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
catch (IllegalAccessException e)
{
throw new IllegalArgumentException("Cannot invoke method: \'"
+ name + "\' @ " + objClazz.getName(), e);
}
catch (InvocationTargetException e)
{
throw new IllegalArgumentException("Invoke method failed: \'"
+ name + "\' @ " + objClazz.getName(),
e.getTargetException());
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException("No such method: \'" + name + "\' @ " + objClazz.getName(), e);
} catch (IllegalAccessException e) {
throw new IllegalArgumentException("Cannot invoke method: \'" + name + "\' @ " + objClazz.getName(), e);
} catch (InvocationTargetException e) {
throw new IllegalArgumentException("Invoke method failed: \'" + name + "\' @ " + objClazz.getName(),
e.getTargetException());
}
}
private static final Object getDeclaredField(Object obj, Class<?> objClazz,
String name)
{
try
{
private static final Object getDeclaredField(Object obj, Class<?> objClazz, String name) {
try {
Field field = objClazz.getDeclaredField(name);
field.setAccessible(true);
return field.get(obj);
}
catch (NoSuchFieldException e)
{
throw new IllegalArgumentException("No such field: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
catch (IllegalAccessException e)
{
throw new IllegalArgumentException("Cannot get field: \'" + name
+ "\' @ " + objClazz.getName(), e);
} catch (NoSuchFieldException e) {
throw new IllegalArgumentException("No such field: \'" + name + "\' @ " + objClazz.getName(), e);
} catch (IllegalAccessException e) {
throw new IllegalArgumentException("Cannot get field: \'" + name + "\' @ " + objClazz.getName(), e);
}
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final int serverId)
throws IOException
{
public void open(Connection conn, String fileName, final int serverId) throws IOException {
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final int serverId,
boolean nonBlocking) throws IOException
{
public void open(Connection conn, String fileName, final int serverId, boolean nonBlocking) throws IOException {
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final long filePosition,
final int serverId) throws IOException
{
public void open(Connection conn, String fileName, final long filePosition, final int serverId) throws IOException {
open(conn, fileName, filePosition, serverId, false);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, long filePosition,
final int serverId, boolean nonBlocking) throws IOException
{
try
{
public void open(Connection conn, String fileName, long filePosition, final int serverId, boolean nonBlocking)
throws IOException {
try {
this.conn = conn;
Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
Object unwrapConn = unwrapConnection(conn, connClazz);
if (unwrapConn == null)
{
throw new IOException("Unable to unwrap "
+ conn.getClass().getName()
+ " to com.mysql.jdbc.ConnectionImpl");
if (unwrapConn == null) {
throw new IOException("Unable to unwrap " + conn.getClass().getName()
+ " to com.mysql.jdbc.ConnectionImpl");
}
// Get underlying IO streams for network communications.
Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
if (connIo == null)
{
throw new IOException("Get null field:"
+ conn.getClass().getName() + "#io");
if (connIo == null) {
throw new IOException("Get null field:" + conn.getClass().getName() + "#io");
}
mysqlOutput = (OutputStream) getDeclaredField(connIo,
connIo.getClass(), "mysqlOutput");
mysqlInput = (InputStream) getDeclaredField(connIo,
connIo.getClass(), "mysqlInput");
mysqlOutput = (OutputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlOutput");
mysqlInput = (InputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlInput");
if (filePosition == 0)
filePosition = BIN_LOG_HEADER_SIZE;
if (filePosition == 0) filePosition = BIN_LOG_HEADER_SIZE;
sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
position = 0;
}
catch (IOException e)
{
} catch (IOException e) {
close(); /* Do cleanup */
logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
+ ", position = " + filePosition);
logger.error("Error on COM_BINLOG_DUMP: file = " + fileName + ", position = " + filePosition);
throw e;
}
catch (ClassNotFoundException e)
{
} catch (ClassNotFoundException e) {
close(); /* Do cleanup */
throw new IOException(
"Unable to load com.mysql.jdbc.ConnectionImpl", e);
throw new IOException("Unable to load com.mysql.jdbc.ConnectionImpl", e);
}
}
@@ -265,8 +202,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param b the byte to put in the buffer
*/
protected final void putByte(byte b)
{
protected final void putByte(byte b) {
ensureCapacity(position + 1);
buffer[position++] = b;
@@ -277,8 +213,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param i16 the integer to put in the buffer
*/
protected final void putInt16(int i16)
{
protected final void putInt16(int i16) {
ensureCapacity(position + 2);
byte[] buf = buffer;
@@ -291,8 +226,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param i32 the integer to put in the buffer
*/
protected final void putInt32(long i32)
{
protected final void putInt32(long i32) {
ensureCapacity(position + 4);
byte[] buf = buffer;
@@ -307,8 +241,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param s the value to put in the buffer
*/
protected final void putString(String s)
{
protected final void putString(String s) {
ensureCapacity(position + (s.length() * 2) + 1);
System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
@@ -316,10 +249,8 @@ public final class DirectLogFetcher extends LogFetcher
buffer[position++] = 0;
}
protected final void sendBinlogDump(String fileName,
final long filePosition, final int serverId, boolean nonBlocking)
throws IOException
{
protected final void sendBinlogDump(String fileName, final long filePosition, final int serverId,
boolean nonBlocking) throws IOException {
position = NET_HEADER_SIZE;
putByte(COM_BINLOG_DUMP);
@@ -327,7 +258,7 @@ public final class DirectLogFetcher extends LogFetcher
int binlog_flags = nonBlocking ? BINLOG_DUMP_NON_BLOCK : 0;
binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
putInt16(binlog_flags); // binlog_flags
putInt32(serverId); // slave's server-id
putInt32(serverId); // slave's server-id
putString(fileName);
final byte[] buf = buffer;
@@ -345,13 +276,10 @@ public final class DirectLogFetcher extends LogFetcher
*
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
*/
public boolean fetch() throws IOException
{
try
{
public boolean fetch() throws IOException {
try {
// Fetching packet header from input.
if (!fetch0(0, NET_HEADER_SIZE))
{
if (!fetch0(0, NET_HEADER_SIZE)) {
logger.warn("Reached end of input stream while fetching header");
return false;
}
@@ -359,61 +287,48 @@ public final class DirectLogFetcher extends LogFetcher
// Fetching the first packet(may a multi-packet).
int netlen = getUint24(PACKET_LEN_OFFSET);
int netnum = getUint8(PACKET_SEQ_OFFSET);
if (!fetch0(NET_HEADER_SIZE, netlen))
{
logger.warn("Reached end of input stream: packet #" + netnum
+ ", len = " + netlen);
if (!fetch0(NET_HEADER_SIZE, netlen)) {
logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
return false;
}
// Detecting error code.
final int mark = getUint8(NET_HEADER_SIZE);
if (mark != 0)
{
if (mark != 0) {
if (mark == 255) // error from master
{
// Indicates an error, for example trying to fetch from wrong
// Indicates an error, for example trying to fetch from
// wrong
// binlog position.
position = NET_HEADER_SIZE + 1;
final int errno = getInt16();
String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
String errmsg = getFixString(limit - position);
throw new IOException("Received error packet:"
+ " errno = " + errno + ", sqlstate = " + sqlstate
+ " errmsg = " + errmsg);
}
else if (mark == 254)
{
throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
+ " errmsg = " + errmsg);
} else if (mark == 254) {
// Indicates end of stream. It's not clear when this would
// be sent.
logger.warn("Received EOF packet from server, apparent"
+ " master disconnected.");
logger.warn("Received EOF packet from server, apparent" + " master disconnected.");
return false;
}
else
{
} else {
// Should not happen.
throw new IOException("Unexpected response " + mark
+ " while fetching binlog: packet #" + netnum
+ ", len = " + netlen);
throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
+ ", len = " + netlen);
}
}
// The first packet is a multi-packet, concatenate the packets.
while (netlen == MAX_PACKET_LENGTH)
{
if (!fetch0(0, NET_HEADER_SIZE))
{
while (netlen == MAX_PACKET_LENGTH) {
if (!fetch0(0, NET_HEADER_SIZE)) {
logger.warn("Reached end of input stream while fetching header");
return false;
}
netlen = getUint24(PACKET_LEN_OFFSET);
netnum = getUint8(PACKET_SEQ_OFFSET);
if (!fetch0(limit, netlen))
{
logger.warn("Reached end of input stream: packet #"
+ netnum + ", len = " + netlen);
if (!fetch0(limit, netlen)) {
logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
return false;
}
}
@@ -423,43 +338,32 @@ public final class DirectLogFetcher extends LogFetcher
position = origin;
limit -= origin;
return true;
}
catch (SocketTimeoutException e)
{
} catch (SocketTimeoutException e) {
close(); /* Do cleanup */
logger.error("Socket timeout expired, closing connection", e);
throw e;
}
catch (InterruptedIOException e)
{
} catch (InterruptedIOException e) {
close(); /* Do cleanup */
logger.warn("I/O interrupted while reading from client socket", e);
throw e;
}
catch (IOException e)
{
} catch (IOException e) {
close(); /* Do cleanup */
logger.error("I/O error while reading from client socket", e);
throw e;
}
}
private final boolean fetch0(final int off, final int len)
throws IOException
{
private final boolean fetch0(final int off, final int len) throws IOException {
ensureCapacity(off + len);
for (int count, n = 0; n < len; n += count)
{
if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
{
for (int count, n = 0; n < len; n += count) {
if (0 > (count = mysqlInput.read(buffer, off + n, len - n))) {
// Reached end of input stream
return false;
}
}
if (limit < off + len)
limit = off + len;
if (limit < off + len) limit = off + len;
return true;
}
@@ -468,19 +372,14 @@ public final class DirectLogFetcher extends LogFetcher
*
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
*/
public void close() throws IOException
{
try
{
if (conn != null)
conn.close();
public void close() throws IOException {
try {
if (conn != null) conn.close();
conn = null;
mysqlInput = null;
mysqlOutput = null;
}
catch (SQLException e)
{
} catch (SQLException e) {
logger.warn("Unable to close connection", e);
}
}

1434
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java
View File

File diff suppressed because it is too large Load Diff

39
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java
View File

@@ -7,69 +7,56 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
/**
* TODO: Document Me!!
*
* NOTE: Log context will NOT write multi-threaded.
* TODO: Document Me!! NOTE: Log context will NOT write multi-threaded.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class LogContext
{
public final class LogContext {
private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
private FormatDescriptionLogEvent formatDescription;
private LogPosition logPosition;
public LogContext()
{
public LogContext(){
this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
}
public LogContext(FormatDescriptionLogEvent descriptionEvent)
{
public LogContext(FormatDescriptionLogEvent descriptionEvent){
this.formatDescription = descriptionEvent;
}
public final LogPosition getLogPosition()
{
public final LogPosition getLogPosition() {
return logPosition;
}
public final void setLogPosition(LogPosition logPosition)
{
public final void setLogPosition(LogPosition logPosition) {
this.logPosition = logPosition;
}
public final FormatDescriptionLogEvent getFormatDescription()
{
public final FormatDescriptionLogEvent getFormatDescription() {
return formatDescription;
}
public final void setFormatDescription(
FormatDescriptionLogEvent formatDescription)
{
public final void setFormatDescription(FormatDescriptionLogEvent formatDescription) {
this.formatDescription = formatDescription;
}
public final void putTable(TableMapLogEvent mapEvent)
{
public final void putTable(TableMapLogEvent mapEvent) {
mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
}
public final TableMapLogEvent getTable(final long tableId)
{
public final TableMapLogEvent getTable(final long tableId) {
return mapOfTable.get(Long.valueOf(tableId));
}
public final void clearAllTables()
{
public final void clearAllTables() {
mapOfTable.clear();
}
public void reset()
{
public void reset() {
formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
mapOfTable.clear();

541
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
View File

@@ -61,28 +61,24 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class LogDecoder
{
public final class LogDecoder {
protected static final Log logger = LogFactory.getLog(LogDecoder.class);
protected final BitSet handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
public LogDecoder()
{
public LogDecoder(){
}
public LogDecoder(final int fromIndex, final int toIndex)
{
public LogDecoder(final int fromIndex, final int toIndex){
handleSet.set(fromIndex, toIndex);
}
public final void handle(final int fromIndex, final int toIndex)
{
public final void handle(final int fromIndex, final int toIndex) {
handleSet.set(fromIndex, toIndex);
}
public final void handle(final int flagIndex)
{
public final void handle(final int flagIndex) {
handleSet.set(flagIndex);
}
@@ -90,48 +86,32 @@ public final class LogDecoder
* Decoding an event from binary-log buffer.
*
* @return <code>UknownLogEvent</code> if event type is unknown or skipped,
* <code>null</code> if buffer is not including a full event.
* <code>null</code> if buffer is not including a full event.
*/
public LogEvent decode(LogBuffer buffer, LogContext context)
throws IOException
{
public LogEvent decode(LogBuffer buffer, LogContext context) throws IOException {
final int limit = buffer.limit();
if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
{
LogHeader header = new LogHeader(buffer,
context.getFormatDescription());
if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
LogHeader header = new LogHeader(buffer, context.getFormatDescription());
final int len = header.getEventLen();
if (limit >= len)
{
if (limit >= len) {
LogEvent event;
/* Checking binary-log's header */
if (handleSet.get(header.getType()))
{
if (handleSet.get(header.getType())) {
buffer.limit(len);
try
{
try {
/* Decoding binary-log to event */
event = decode(buffer, header, context);
}
catch (IOException e)
{
if (logger.isWarnEnabled())
logger.warn("Decoding "
+ LogEvent.getTypeName(header.getType())
+ " failed from: "
+ context.getLogPosition(), e);
} catch (IOException e) {
if (logger.isWarnEnabled()) logger.warn("Decoding " + LogEvent.getTypeName(header.getType())
+ " failed from: " + context.getLogPosition(), e);
throw e;
}
finally
{
} finally {
buffer.limit(limit); /* Restore limit */
}
}
else
{
} else {
/* Ignore unsupported binary-log. */
event = new UnknownLogEvent(header);
}
@@ -152,338 +132,275 @@ public final class LogDecoder
*
* @return <code>UknownLogEvent</code> if event type is unknown or skipped.
*/
public static LogEvent decode(LogBuffer buffer, LogHeader header,
LogContext context) throws IOException
{
public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext context) throws IOException {
FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
LogPosition logPosition = context.getLogPosition();
int checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
if (header.getType() != LogEvent.FORMAT_DESCRIPTION_EVENT) {
checksumAlg = descriptionEvent.header.getChecksumAlg();
}else {
} else {
// 如果是format事件自己，也需要处理checksum
checksumAlg = header.getChecksumAlg();
}
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
// remove checksum bytes
buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
}
switch (header.getType())
{
case LogEvent.QUERY_EVENT:
{
QueryLogEvent event = new QueryLogEvent(header, buffer,
descriptionEvent);
switch (header.getType()) {
case LogEvent.QUERY_EVENT: {
QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.XID_EVENT:
{
XidLogEvent event = new XidLogEvent(header, buffer,
descriptionEvent);
case LogEvent.XID_EVENT: {
XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.TABLE_MAP_EVENT:
{
TableMapLogEvent mapEvent = new TableMapLogEvent(header,
buffer, descriptionEvent);
case LogEvent.TABLE_MAP_EVENT: {
TableMapLogEvent mapEvent = new TableMapLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
context.putTable(mapEvent);
return mapEvent;
}
case LogEvent.WRITE_ROWS_EVENT_V1:
{
RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.UPDATE_ROWS_EVENT_V1:
{
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.DELETE_ROWS_EVENT_V1:
{
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.ROTATE_EVENT:
{
RotateLogEvent event = new RotateLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition = new LogPosition(event.getFilename(),
event.getPosition());
context.setLogPosition(logPosition);
return event;
}
case LogEvent.LOAD_EVENT:
case LogEvent.NEW_LOAD_EVENT:
{
LoadLogEvent event = new LoadLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported SLAVE_EVENT from: "
+ context.getLogPosition());
break;
}
case LogEvent.CREATE_FILE_EVENT:
{
CreateFileLogEvent event = new CreateFileLogEvent(header,
buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.APPEND_BLOCK_EVENT:
{
AppendBlockLogEvent event = new AppendBlockLogEvent(header,
buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.DELETE_FILE_EVENT:
{
DeleteFileLogEvent event = new DeleteFileLogEvent(header,
buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.EXEC_LOAD_EVENT:
{
ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.START_EVENT_V3:
{
/* This is sent only by MySQL <=4.x */
StartLogEventV3 event = new StartLogEventV3(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.STOP_EVENT:
{
StopLogEvent event = new StopLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.INTVAR_EVENT:
{
IntvarLogEvent event = new IntvarLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.RAND_EVENT:
{
RandLogEvent event = new RandLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.USER_VAR_EVENT:
{
UserVarLogEvent event = new UserVarLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.FORMAT_DESCRIPTION_EVENT:
{
descriptionEvent = new FormatDescriptionLogEvent(header,
buffer, descriptionEvent);
context.setFormatDescription(descriptionEvent);
return descriptionEvent;
}
case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Write_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Update_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Delete_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.BEGIN_LOAD_QUERY_EVENT:
{
BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
{
ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.INCIDENT_EVENT:
{
IncidentLogEvent event = new IncidentLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.HEARTBEAT_LOG_EVENT:
{
HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.IGNORABLE_LOG_EVENT:
{
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.ROWS_QUERY_LOG_EVENT:
{
RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.WRITE_ROWS_EVENT: {
case LogEvent.WRITE_ROWS_EVENT_V1: {
RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.UPDATE_ROWS_EVENT: {
case LogEvent.UPDATE_ROWS_EVENT_V1: {
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.DELETE_ROWS_EVENT: {
case LogEvent.DELETE_ROWS_EVENT_V1: {
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.GTID_LOG_EVENT:
case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
case LogEvent.ROTATE_EVENT: {
RotateLogEvent event = new RotateLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition = new LogPosition(event.getFilename(), event.getPosition());
context.setLogPosition(logPosition);
return event;
}
case LogEvent.LOAD_EVENT:
case LogEvent.NEW_LOAD_EVENT: {
LoadLogEvent event = new LoadLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
{
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: "
+ context.getLogPosition());
break;
}
case LogEvent.CREATE_FILE_EVENT: {
CreateFileLogEvent event = new CreateFileLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.APPEND_BLOCK_EVENT: {
AppendBlockLogEvent event = new AppendBlockLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.DELETE_FILE_EVENT: {
DeleteFileLogEvent event = new DeleteFileLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.EXEC_LOAD_EVENT: {
ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.START_EVENT_V3: {
/* This is sent only by MySQL <=4.x */
StartLogEventV3 event = new StartLogEventV3(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.STOP_EVENT: {
StopLogEvent event = new StopLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.INTVAR_EVENT: {
IntvarLogEvent event = new IntvarLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.RAND_EVENT: {
RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.USER_VAR_EVENT: {
UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.FORMAT_DESCRIPTION_EVENT: {
descriptionEvent = new FormatDescriptionLogEvent(header, buffer, descriptionEvent);
context.setFormatDescription(descriptionEvent);
return descriptionEvent;
}
case LogEvent.PRE_GA_WRITE_ROWS_EVENT: {
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Write_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: {
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Update_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.PRE_GA_DELETE_ROWS_EVENT: {
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Delete_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.BEGIN_LOAD_QUERY_EVENT: {
BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.EXECUTE_LOAD_QUERY_EVENT: {
ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.INCIDENT_EVENT: {
IncidentLogEvent event = new IncidentLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.HEARTBEAT_LOG_EVENT: {
HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.IGNORABLE_LOG_EVENT: {
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.ROWS_QUERY_LOG_EVENT: {
RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.WRITE_ROWS_EVENT: {
RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.UPDATE_ROWS_EVENT: {
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.DELETE_ROWS_EVENT: {
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.GTID_LOG_EVENT:
case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
{
case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.ANNOTATE_ROWS_EVENT:
{
AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.BINLOG_CHECKPOINT_EVENT:
{
BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.GTID_EVENT:
{
MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.GTID_LIST_EVENT:
{
MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
default:
/*
Create an object of Ignorable_log_event for unrecognized sub-class.
So that SLAVE SQL THREAD will only update the position and continue.
*/
if((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0){
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
case LogEvent.ANNOTATE_ROWS_EVENT: {
AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}else {
if (logger.isWarnEnabled())
logger.warn("Skipping unrecognized binlog event "
+ LogEvent.getTypeName(header.getType()) + " from: "
+ context.getLogPosition());
}
case LogEvent.BINLOG_CHECKPOINT_EVENT: {
BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.GTID_EVENT: {
MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.GTID_LIST_EVENT: {
MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
default:
/*
* Create an object of Ignorable_log_event for unrecognized
* sub-class. So that SLAVE SQL THREAD will only update the
* position and continue.
*/
if ((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0) {
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
} else {
if (logger.isWarnEnabled()) logger.warn("Skipping unrecognized binlog event "
+ LogEvent.getTypeName(header.getType()) + " from: "
+ context.getLogPosition());
}
}
/* updating position in context */

487
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java
View File

@@ -7,239 +7,218 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
/**
* Binary log event definitions. This includes generic code common to all types
* of log events, as well as specific code for each type of log event.
*
* - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
* in little endian, i.e., the least significant byte first, unless otherwise
* specified.
*
* representation of unsigned integers, called Packed Integer. A Packed Integer
* has the capacity of storing up to 8-byte integers, while small integers still
* can use 1, 3, or 4 bytes. The value of the first byte determines how to read
* the number, according to the following table:
*
* of log events, as well as specific code for each type of log event. - All
* numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored in
* little endian, i.e., the least significant byte first, unless otherwise
* specified. representation of unsigned integers, called Packed Integer. A
* Packed Integer has the capacity of storing up to 8-byte integers, while small
* integers still can use 1, 3, or 4 bytes. The value of the first byte
* determines how to read the number, according to the following table:
* <table>
* <caption>Format of Packed Integer</caption>
*
* <tr>
* <th>First byte</th>
* <th>Format</th>
* </tr>
*
* <tr>
* <td>0-250</td>
* <td>The first byte is the number (in the range 0-250), and no more bytes are
* used.</td>
* </tr>
*
* <tr>
* <td>252</td>
* <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
* </tr>
*
* <tr>
* <td>253</td>
* <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
* </tr>
*
* <tr>
* <td>254</td>
* <td>Eight more bytes are used. The number is in the range
* 0xffffff-0xffffffffffffffff.</td>
* </tr>
*
* </table>
*
* - Strings are stored in various formats. The format of each string is
* documented separately.
*
* @see mysql-5.1.60/sql/log_event.h
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public abstract class LogEvent
{
public abstract class LogEvent {
/*
* 3 is MySQL 4.x; 4 is MySQL 5.0.0.
* Compared to version 3, version 4 has:
* - a different Start_log_event, which includes info about the binary log
* 3 is MySQL 4.x; 4 is MySQL 5.0.0. Compared to version 3, version 4 has: -
* a different Start_log_event, which includes info about the binary log
* (sizes of headers); this info is included for better compatibility if the
* master's MySQL version is different from the slave's.
* - all events have a unique ID (the triplet (server_id, timestamp at server
* start, other) to be sure an event is not executed more than once in a
* multimaster setup, example:
* M1
* / \
* v v
* M2 M3
* \ /
* v v
* S
* if a query is run on M1, it will arrive twice on S, so we need that S
* remembers the last unique ID it has processed, to compare and know if the
* event should be skipped or not. Example of ID: we already have the server id
* (4 bytes), plus:
* master's MySQL version is different from the slave's. - all events have a
* unique ID (the triplet (server_id, timestamp at server start, other) to
* be sure an event is not executed more than once in a multimaster setup,
* example: M1 / \ v v M2 M3 \ / v v S if a query is run on M1, it will
* arrive twice on S, so we need that S remembers the last unique ID it has
* processed, to compare and know if the event should be skipped or not.
* Example of ID: we already have the server id (4 bytes), plus:
* timestamp_when_the_master_started (4 bytes), a counter (a sequence number
* which increments every time we write an event to the binlog) (3 bytes).
* Q: how do we handle when the counter is overflowed and restarts from 0 ?
*
* - Query and Load (Create or Execute) events may have a more precise
* timestamp (with microseconds), number of matched/affected/warnings rows
* and fields of session variables: SQL_MODE,
* FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
* charsets, the PASSWORD() version (old/new/...).
* timestamp (with microseconds), number of matched/affected/warnings rows
* and fields of session variables: SQL_MODE, FOREIGN_KEY_CHECKS,
* UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and charsets, the
* PASSWORD() version (old/new/...).
*/
public static final int BINLOG_VERSION = 4;
public static final int BINLOG_VERSION = 4;
/* Default 5.0 server version */
public static final String SERVER_VERSION = "5.0";
public static final String SERVER_VERSION = "5.0";
/**
* Event header offsets; these point to places inside the fixed header.
*/
public static final int EVENT_TYPE_OFFSET = 4;
public static final int SERVER_ID_OFFSET = 5;
public static final int EVENT_LEN_OFFSET = 9;
public static final int LOG_POS_OFFSET = 13;
public static final int FLAGS_OFFSET = 17;
public static final int EVENT_TYPE_OFFSET = 4;
public static final int SERVER_ID_OFFSET = 5;
public static final int EVENT_LEN_OFFSET = 9;
public static final int LOG_POS_OFFSET = 13;
public static final int FLAGS_OFFSET = 17;
/* event-specific post-header sizes */
// where 3.23, 4.x and 5.0 agree
public static final int QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2);
public static final int QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2);
// where 5.0 differs: 2 for len of N-bytes vars.
public static final int QUERY_HEADER_LEN = (QUERY_HEADER_MINIMAL_LEN + 2);
public static final int QUERY_HEADER_LEN = (QUERY_HEADER_MINIMAL_LEN + 2);
/* Enumeration type for the different types of log events. */
public static final int UNKNOWN_EVENT = 0;
public static final int START_EVENT_V3 = 1;
public static final int QUERY_EVENT = 2;
public static final int STOP_EVENT = 3;
public static final int ROTATE_EVENT = 4;
public static final int INTVAR_EVENT = 5;
public static final int LOAD_EVENT = 6;
public static final int SLAVE_EVENT = 7;
public static final int CREATE_FILE_EVENT = 8;
public static final int APPEND_BLOCK_EVENT = 9;
public static final int EXEC_LOAD_EVENT = 10;
public static final int DELETE_FILE_EVENT = 11;
public static final int UNKNOWN_EVENT = 0;
public static final int START_EVENT_V3 = 1;
public static final int QUERY_EVENT = 2;
public static final int STOP_EVENT = 3;
public static final int ROTATE_EVENT = 4;
public static final int INTVAR_EVENT = 5;
public static final int LOAD_EVENT = 6;
public static final int SLAVE_EVENT = 7;
public static final int CREATE_FILE_EVENT = 8;
public static final int APPEND_BLOCK_EVENT = 9;
public static final int EXEC_LOAD_EVENT = 10;
public static final int DELETE_FILE_EVENT = 11;
/**
* NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer sql_ex,
* allowing multibyte TERMINATED BY etc; both types share the same class
* (Load_log_event)
*/
public static final int NEW_LOAD_EVENT = 12;
public static final int RAND_EVENT = 13;
public static final int USER_VAR_EVENT = 14;
public static final int FORMAT_DESCRIPTION_EVENT = 15;
public static final int XID_EVENT = 16;
public static final int BEGIN_LOAD_QUERY_EVENT = 17;
public static final int EXECUTE_LOAD_QUERY_EVENT = 18;
public static final int NEW_LOAD_EVENT = 12;
public static final int RAND_EVENT = 13;
public static final int USER_VAR_EVENT = 14;
public static final int FORMAT_DESCRIPTION_EVENT = 15;
public static final int XID_EVENT = 16;
public static final int BEGIN_LOAD_QUERY_EVENT = 17;
public static final int EXECUTE_LOAD_QUERY_EVENT = 18;
public static final int TABLE_MAP_EVENT = 19;
public static final int TABLE_MAP_EVENT = 19;
/**
* These event numbers were used for 5.1.0 to 5.1.15 and are therefore
* obsolete.
*/
public static final int PRE_GA_WRITE_ROWS_EVENT = 20;
public static final int PRE_GA_UPDATE_ROWS_EVENT = 21;
public static final int PRE_GA_DELETE_ROWS_EVENT = 22;
public static final int PRE_GA_WRITE_ROWS_EVENT = 20;
public static final int PRE_GA_UPDATE_ROWS_EVENT = 21;
public static final int PRE_GA_DELETE_ROWS_EVENT = 22;
/**
* These event numbers are used from 5.1.16 and forward
*/
public static final int WRITE_ROWS_EVENT_V1 = 23;
public static final int UPDATE_ROWS_EVENT_V1 = 24;
public static final int DELETE_ROWS_EVENT_V1 = 25;
public static final int WRITE_ROWS_EVENT_V1 = 23;
public static final int UPDATE_ROWS_EVENT_V1 = 24;
public static final int DELETE_ROWS_EVENT_V1 = 25;
/**
* Something out of the ordinary happened on the master
*/
public static final int INCIDENT_EVENT = 26;
public static final int INCIDENT_EVENT = 26;
/**
* Heartbeat event to be send by master at its idle time to ensure master's online status to slave
* Heartbeat event to be send by master at its idle time to ensure master's
* online status to slave
*/
public static final int HEARTBEAT_LOG_EVENT = 27;
public static final int HEARTBEAT_LOG_EVENT = 27;
/**
* In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
* case there is code for handling it, but which can be ignored if it is not recognized.
* In some situations, it is necessary to send over ignorable data to the
* slave: data that a slave can handle in case there is code for handling
* it, but which can be ignored if it is not recognized.
*/
public static final int IGNORABLE_LOG_EVENT = 28;
public static final int ROWS_QUERY_LOG_EVENT = 29;
public static final int IGNORABLE_LOG_EVENT = 28;
public static final int ROWS_QUERY_LOG_EVENT = 29;
/** Version 2 of the Row events */
public static final int WRITE_ROWS_EVENT = 30;
public static final int UPDATE_ROWS_EVENT = 31;
public static final int DELETE_ROWS_EVENT = 32;
public static final int WRITE_ROWS_EVENT = 30;
public static final int UPDATE_ROWS_EVENT = 31;
public static final int DELETE_ROWS_EVENT = 32;
public static final int GTID_LOG_EVENT = 33;
public static final int ANONYMOUS_GTID_LOG_EVENT = 34;
public static final int GTID_LOG_EVENT = 33;
public static final int ANONYMOUS_GTID_LOG_EVENT = 34;
public static final int PREVIOUS_GTIDS_LOG_EVENT = 35;
public static final int PREVIOUS_GTIDS_LOG_EVENT = 35;
// mariaDb 5.5.34
/* New MySQL/Sun events are to be added right above this comment */
public static final int MYSQL_EVENTS_END = 36;
public static final int MYSQL_EVENTS_END = 36;
public static final int MARIA_EVENTS_BEGIN = 160;
public static final int MARIA_EVENTS_BEGIN = 160;
/* New Maria event numbers start from here */
public static final int ANNOTATE_ROWS_EVENT = 160;
public static final int ANNOTATE_ROWS_EVENT = 160;
/*
Binlog checkpoint event. Used for XA crash recovery on the master, not used
in replication.
A binlog checkpoint event specifies a binlog file such that XA crash
recovery can start from that file - and it is guaranteed to find all XIDs
that are prepared in storage engines but not yet committed.
* Binlog checkpoint event. Used for XA crash recovery on the master, not
* used in replication. A binlog checkpoint event specifies a binlog file
* such that XA crash recovery can start from that file - and it is
* guaranteed to find all XIDs that are prepared in storage engines but not
* yet committed.
*/
public static final int BINLOG_CHECKPOINT_EVENT = 161;
public static final int BINLOG_CHECKPOINT_EVENT = 161;
/*
Gtid event. For global transaction ID, used to start a new event group,
instead of the old BEGIN query event, and also to mark stand-alone
events.
* Gtid event. For global transaction ID, used to start a new event group,
* instead of the old BEGIN query event, and also to mark stand-alone
* events.
*/
public static final int GTID_EVENT = 162;
public static final int GTID_EVENT = 162;
/*
Gtid list event. Logged at the start of every binlog, to record the
current replication state. This consists of the last GTID seen for
each replication domain.
* Gtid list event. Logged at the start of every binlog, to record the
* current replication state. This consists of the last GTID seen for each
* replication domain.
*/
public static final int GTID_LIST_EVENT = 163;
public static final int GTID_LIST_EVENT = 163;
/** end marker */
public static final int ENUM_END_EVENT = 164;
public static final int ENUM_END_EVENT = 164;
/**
1 byte length, 1 byte format
Length is total length in bytes, including 2 byte header
Length values 0 and 1 are currently invalid and reserved.
*/
public static final int EXTRA_ROW_INFO_LEN_OFFSET = 0;
public static final int EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
public static final int EXTRA_ROW_INFO_HDR_BYTES = 2;
public static final int EXTRA_ROW_INFO_MAX_PAYLOAD = (255 - EXTRA_ROW_INFO_HDR_BYTES);
* 1 byte length, 1 byte format Length is total length in bytes, including 2
* byte header Length values 0 and 1 are currently invalid and reserved.
*/
public static final int EXTRA_ROW_INFO_LEN_OFFSET = 0;
public static final int EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
public static final int EXTRA_ROW_INFO_HDR_BYTES = 2;
public static final int EXTRA_ROW_INFO_MAX_PAYLOAD = (255 - EXTRA_ROW_INFO_HDR_BYTES);
// Events are without checksum though its generator
public static final int BINLOG_CHECKSUM_ALG_OFF = 0;
public static final int BINLOG_CHECKSUM_ALG_OFF = 0;
// is checksum-capable New Master (NM).
// CRC32 of zlib algorithm.
public static final int BINLOG_CHECKSUM_ALG_CRC32 = 1;
public static final int BINLOG_CHECKSUM_ALG_CRC32 = 1;
// the cut line: valid alg range is [1, 0x7f].
public static final int BINLOG_CHECKSUM_ALG_ENUM_END = 2;
public static final int BINLOG_CHECKSUM_ALG_ENUM_END = 2;
// special value to tag undetermined yet checksum
public static final int BINLOG_CHECKSUM_ALG_UNDEF = 255;
public static final int BINLOG_CHECKSUM_ALG_UNDEF = 255;
// or events from checksum-unaware servers
public static final int CHECKSUM_CRC32_SIGNATURE_LEN = 4;
public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
public static final int CHECKSUM_CRC32_SIGNATURE_LEN = 4;
public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
/**
* defined statically while there is just one alg implemented
*/
public static final int BINLOG_CHECKSUM_LEN = CHECKSUM_CRC32_SIGNATURE_LEN;
public static final int BINLOG_CHECKSUM_LEN = CHECKSUM_CRC32_SIGNATURE_LEN;
/* MySQL or old MariaDB slave with no announced capability. */
public static final int MARIA_SLAVE_CAPABILITY_UNKNOWN = 0;
@@ -258,124 +237,121 @@ public abstract class LogEvent
/* Our capability. */
public static final int MARIA_SLAVE_CAPABILITY_MINE = MARIA_SLAVE_CAPABILITY_GTID;
/**
For an event, 'e', carrying a type code, that a slave,
's', does not recognize, 's' will check 'e' for
LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
is ignored. Otherwise, 's' acknowledges that it has
found an unknown event in the relay log.
*/
public static final int LOG_EVENT_IGNORABLE_F = 0x80;
/** enum_field_types */
public static final int MYSQL_TYPE_DECIMAL = 0;
public static final int MYSQL_TYPE_TINY = 1;
public static final int MYSQL_TYPE_SHORT = 2;
public static final int MYSQL_TYPE_LONG = 3;
public static final int MYSQL_TYPE_FLOAT = 4;
public static final int MYSQL_TYPE_DOUBLE = 5;
public static final int MYSQL_TYPE_NULL = 6;
public static final int MYSQL_TYPE_TIMESTAMP = 7;
public static final int MYSQL_TYPE_LONGLONG = 8;
public static final int MYSQL_TYPE_INT24 = 9;
public static final int MYSQL_TYPE_DATE = 10;
public static final int MYSQL_TYPE_TIME = 11;
public static final int MYSQL_TYPE_DATETIME = 12;
public static final int MYSQL_TYPE_YEAR = 13;
public static final int MYSQL_TYPE_NEWDATE = 14;
public static final int MYSQL_TYPE_VARCHAR = 15;
public static final int MYSQL_TYPE_BIT = 16;
public static final int MYSQL_TYPE_TIMESTAMP2 = 17;
public static final int MYSQL_TYPE_DATETIME2 = 18;
public static final int MYSQL_TYPE_TIME2 = 19;
public static final int MYSQL_TYPE_NEWDECIMAL = 246;
public static final int MYSQL_TYPE_ENUM = 247;
public static final int MYSQL_TYPE_SET = 248;
public static final int MYSQL_TYPE_TINY_BLOB = 249;
public static final int MYSQL_TYPE_MEDIUM_BLOB = 250;
public static final int MYSQL_TYPE_LONG_BLOB = 251;
public static final int MYSQL_TYPE_BLOB = 252;
public static final int MYSQL_TYPE_VAR_STRING = 253;
public static final int MYSQL_TYPE_STRING = 254;
public static final int MYSQL_TYPE_GEOMETRY = 255;
public static String getTypeName(final int type)
{
switch (type)
{
case START_EVENT_V3:
return "Start_v3";
case STOP_EVENT:
return "Stop";
case QUERY_EVENT:
return "Query";
case ROTATE_EVENT:
return "Rotate";
case INTVAR_EVENT:
return "Intvar";
case LOAD_EVENT:
return "Load";
case NEW_LOAD_EVENT:
return "New_load";
case SLAVE_EVENT:
return "Slave";
case CREATE_FILE_EVENT:
return "Create_file";
case APPEND_BLOCK_EVENT:
return "Append_block";
case DELETE_FILE_EVENT:
return "Delete_file";
case EXEC_LOAD_EVENT:
return "Exec_load";
case RAND_EVENT:
return "RAND";
case XID_EVENT:
return "Xid";
case USER_VAR_EVENT:
return "User var";
case FORMAT_DESCRIPTION_EVENT:
return "Format_desc";
case TABLE_MAP_EVENT:
return "Table_map";
case PRE_GA_WRITE_ROWS_EVENT:
return "Write_rows_event_old";
case PRE_GA_UPDATE_ROWS_EVENT:
return "Update_rows_event_old";
case PRE_GA_DELETE_ROWS_EVENT:
return "Delete_rows_event_old";
case WRITE_ROWS_EVENT_V1:
return "Write_rows_v1";
case UPDATE_ROWS_EVENT_V1:
return "Update_rows_v1";
case DELETE_ROWS_EVENT_V1:
return "Delete_rows_v1";
case BEGIN_LOAD_QUERY_EVENT:
return "Begin_load_query";
case EXECUTE_LOAD_QUERY_EVENT:
return "Execute_load_query";
case INCIDENT_EVENT:
return "Incident";
case HEARTBEAT_LOG_EVENT:
return "Heartbeat";
case IGNORABLE_LOG_EVENT:
return "Ignorable";
case ROWS_QUERY_LOG_EVENT:
return "Rows_query";
case WRITE_ROWS_EVENT:
return "Write_rows";
case UPDATE_ROWS_EVENT:
return "Update_rows";
case DELETE_ROWS_EVENT:
return "Delete_rows";
case GTID_LOG_EVENT:
return "Gtid";
case ANONYMOUS_GTID_LOG_EVENT:
return "Anonymous_Gtid";
case PREVIOUS_GTIDS_LOG_EVENT:
return "Previous_gtids";
default:
return "Unknown"; /* impossible */
/**
* For an event, 'e', carrying a type code, that a slave, 's', does not
* recognize, 's' will check 'e' for LOG_EVENT_IGNORABLE_F, and if the flag
* is set, then 'e' is ignored. Otherwise, 's' acknowledges that it has
* found an unknown event in the relay log.
*/
public static final int LOG_EVENT_IGNORABLE_F = 0x80;
/** enum_field_types */
public static final int MYSQL_TYPE_DECIMAL = 0;
public static final int MYSQL_TYPE_TINY = 1;
public static final int MYSQL_TYPE_SHORT = 2;
public static final int MYSQL_TYPE_LONG = 3;
public static final int MYSQL_TYPE_FLOAT = 4;
public static final int MYSQL_TYPE_DOUBLE = 5;
public static final int MYSQL_TYPE_NULL = 6;
public static final int MYSQL_TYPE_TIMESTAMP = 7;
public static final int MYSQL_TYPE_LONGLONG = 8;
public static final int MYSQL_TYPE_INT24 = 9;
public static final int MYSQL_TYPE_DATE = 10;
public static final int MYSQL_TYPE_TIME = 11;
public static final int MYSQL_TYPE_DATETIME = 12;
public static final int MYSQL_TYPE_YEAR = 13;
public static final int MYSQL_TYPE_NEWDATE = 14;
public static final int MYSQL_TYPE_VARCHAR = 15;
public static final int MYSQL_TYPE_BIT = 16;
public static final int MYSQL_TYPE_TIMESTAMP2 = 17;
public static final int MYSQL_TYPE_DATETIME2 = 18;
public static final int MYSQL_TYPE_TIME2 = 19;
public static final int MYSQL_TYPE_NEWDECIMAL = 246;
public static final int MYSQL_TYPE_ENUM = 247;
public static final int MYSQL_TYPE_SET = 248;
public static final int MYSQL_TYPE_TINY_BLOB = 249;
public static final int MYSQL_TYPE_MEDIUM_BLOB = 250;
public static final int MYSQL_TYPE_LONG_BLOB = 251;
public static final int MYSQL_TYPE_BLOB = 252;
public static final int MYSQL_TYPE_VAR_STRING = 253;
public static final int MYSQL_TYPE_STRING = 254;
public static final int MYSQL_TYPE_GEOMETRY = 255;
public static String getTypeName(final int type) {
switch (type) {
case START_EVENT_V3:
return "Start_v3";
case STOP_EVENT:
return "Stop";
case QUERY_EVENT:
return "Query";
case ROTATE_EVENT:
return "Rotate";
case INTVAR_EVENT:
return "Intvar";
case LOAD_EVENT:
return "Load";
case NEW_LOAD_EVENT:
return "New_load";
case SLAVE_EVENT:
return "Slave";
case CREATE_FILE_EVENT:
return "Create_file";
case APPEND_BLOCK_EVENT:
return "Append_block";
case DELETE_FILE_EVENT:
return "Delete_file";
case EXEC_LOAD_EVENT:
return "Exec_load";
case RAND_EVENT:
return "RAND";
case XID_EVENT:
return "Xid";
case USER_VAR_EVENT:
return "User var";
case FORMAT_DESCRIPTION_EVENT:
return "Format_desc";
case TABLE_MAP_EVENT:
return "Table_map";
case PRE_GA_WRITE_ROWS_EVENT:
return "Write_rows_event_old";
case PRE_GA_UPDATE_ROWS_EVENT:
return "Update_rows_event_old";
case PRE_GA_DELETE_ROWS_EVENT:
return "Delete_rows_event_old";
case WRITE_ROWS_EVENT_V1:
return "Write_rows_v1";
case UPDATE_ROWS_EVENT_V1:
return "Update_rows_v1";
case DELETE_ROWS_EVENT_V1:
return "Delete_rows_v1";
case BEGIN_LOAD_QUERY_EVENT:
return "Begin_load_query";
case EXECUTE_LOAD_QUERY_EVENT:
return "Execute_load_query";
case INCIDENT_EVENT:
return "Incident";
case HEARTBEAT_LOG_EVENT:
return "Heartbeat";
case IGNORABLE_LOG_EVENT:
return "Ignorable";
case ROWS_QUERY_LOG_EVENT:
return "Rows_query";
case WRITE_ROWS_EVENT:
return "Write_rows";
case UPDATE_ROWS_EVENT:
return "Update_rows";
case DELETE_ROWS_EVENT:
return "Delete_rows";
case GTID_LOG_EVENT:
return "Gtid";
case ANONYMOUS_GTID_LOG_EVENT:
return "Anonymous_Gtid";
case PREVIOUS_GTIDS_LOG_EVENT:
return "Previous_gtids";
default:
return "Unknown"; /* impossible */
}
}
@@ -383,16 +359,14 @@ public abstract class LogEvent
protected final LogHeader header;
protected LogEvent(LogHeader header)
{
protected LogEvent(LogHeader header){
this.header = header;
}
/**
* Return event header.
*/
public final LogHeader getHeader()
{
public final LogHeader getHeader() {
return header;
}
@@ -400,16 +374,14 @@ public abstract class LogEvent
* The total size of this event, in bytes. In other words, this is the sum
* of the sizes of Common-Header, Post-Header, and Body.
*/
public final int getEventLen()
{
public final int getEventLen() {
return header.getEventLen();
}
/**
* Server ID of the server that created the event.
*/
public final long getServerId()
{
public final long getServerId() {
return header.getServerId();
}
@@ -420,18 +392,15 @@ public abstract class LogEvent
* file. In a relay log, this is the position of the next event in the
* master's binlog.
*/
public final long getLogPos()
{
public final long getLogPos() {
return header.getLogPos();
}
/**
* The time when the query started, in seconds since 1970.
*/
public final long getWhen()
{
public final long getWhen() {
return header.getWhen();
}
}

22
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java
View File

@@ -29,8 +29,8 @@ import java.util.Arrays;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public abstract class LogFetcher extends LogBuffer implements Closeable
{
public abstract class LogFetcher extends LogBuffer implements Closeable {
/** Default initial capacity. */
public static final int DEFAULT_INITIAL_CAPACITY = 8192;
@@ -42,18 +42,15 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
protected final float factor;
public LogFetcher()
{
public LogFetcher(){
this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
}
public LogFetcher(final int initialCapacity)
{
public LogFetcher(final int initialCapacity){
this(initialCapacity, DEFAULT_GROWTH_FACTOR);
}
public LogFetcher(final int initialCapacity, final float growthFactor)
{
public LogFetcher(final int initialCapacity, final float growthFactor){
this.buffer = new byte[initialCapacity];
this.factor = growthFactor;
}
@@ -65,15 +62,12 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
*
* @param minCapacity the desired minimum capacity
*/
protected final void ensureCapacity(final int minCapacity)
{
protected final void ensureCapacity(final int minCapacity) {
final int oldCapacity = buffer.length;
if (minCapacity > oldCapacity)
{
if (minCapacity > oldCapacity) {
int newCapacity = (int) (oldCapacity * factor);
if (newCapacity < minCapacity)
newCapacity = minCapacity;
if (newCapacity < minCapacity) newCapacity = minCapacity;
buffer = Arrays.copyOf(buffer, newCapacity);
}

53
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java
View File

@@ -6,8 +6,8 @@ package com.taobao.tddl.dbsync.binlog;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class LogPosition implements Cloneable, Comparable<LogPosition>
{
public class LogPosition implements Cloneable, Comparable<LogPosition> {
/* binlog file's name */
protected String fileName;
@@ -19,8 +19,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @param fileName file name for binlog files: mysql-bin.000001
*/
public LogPosition(String fileName)
{
public LogPosition(String fileName){
this.fileName = fileName;
this.position = 0L;
}
@@ -30,8 +29,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @param fileName file name for binlog files: mysql-bin.000001
*/
public LogPosition(String fileName, final long position)
{
public LogPosition(String fileName, final long position){
this.fileName = fileName;
this.position = position;
}
@@ -39,31 +37,24 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
/**
* Binlog position copy init.
*/
public LogPosition(LogPosition source)
{
public LogPosition(LogPosition source){
this.fileName = source.fileName;
this.position = source.position;
}
public final String getFileName()
{
public final String getFileName() {
return fileName;
}
public final long getPosition()
{
public final long getPosition() {
return position;
}
/* Clone binlog position without CloneNotSupportedException */
public LogPosition clone()
{
try
{
public LogPosition clone() {
try {
return (LogPosition) super.clone();
}
catch (CloneNotSupportedException e)
{
} catch (CloneNotSupportedException e) {
// Never happend
return null;
}
@@ -72,12 +63,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
/**
* Compares with the specified fileName and position.
*/
public final int compareTo(String fileName, final long position)
{
public final int compareTo(String fileName, final long position) {
final int val = this.fileName.compareTo(fileName);
if (val == 0)
{
if (val == 0) {
return (int) (this.position - position);
}
return val;
@@ -88,12 +77,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
public int compareTo(LogPosition o)
{
public int compareTo(LogPosition o) {
final int val = fileName.compareTo(o.fileName);
if (val == 0)
{
if (val == 0) {
return (int) (position - o.position);
}
return val;
@@ -104,13 +91,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @see java.lang.Object#equals(java.lang.Object)
*/
public boolean equals(Object obj)
{
if (obj instanceof LogPosition)
{
public boolean equals(Object obj) {
if (obj instanceof LogPosition) {
LogPosition pos = ((LogPosition) obj);
return fileName.equals(pos.fileName)
&& (this.position == pos.position);
return fileName.equals(pos.fileName) && (this.position == pos.position);
}
return false;
}
@@ -120,8 +104,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @see java.lang.Object#toString()
*/
public String toString()
{
public String toString() {
return fileName + ':' + position;
}
}

17
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java
View File

@@ -9,8 +9,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class AppendBlockLogEvent extends LogEvent
{
public class AppendBlockLogEvent extends LogEvent {
private final LogBuffer blockBuf;
private final int blockLen;
@@ -19,9 +19,7 @@ public class AppendBlockLogEvent extends LogEvent
/* AB = "Append Block" */
public static final int AB_FILE_ID_OFFSET = 0;
public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -36,18 +34,15 @@ public class AppendBlockLogEvent extends LogEvent
blockBuf = buffer.duplicate(blockLen);
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
public final LogBuffer getBuffer()
{
public final LogBuffer getBuffer() {
return blockBuf;
}
public final byte[] getData()
{
public final byte[] getData() {
return blockBuf.getData();
}
}

8
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java
View File

@@ -10,11 +10,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
{
public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent {
public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

49
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java
View File

@@ -8,8 +8,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class CreateFileLogEvent extends LoadLogEvent
{
public final class CreateFileLogEvent extends LoadLogEvent {
protected LogBuffer blockBuf;
protected int blockLen;
protected long fileId;
@@ -20,9 +20,7 @@ public final class CreateFileLogEvent extends LoadLogEvent
public static final int CF_FILE_ID_OFFSET = 0;
public static final int CF_DATA_OFFSET = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public CreateFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
final int headerLen = descriptionEvent.commonHeaderLen;
@@ -30,44 +28,37 @@ public final class CreateFileLogEvent extends LoadLogEvent
final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
copyLogEvent(buffer,
((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
: (headerLen + loadHeaderLen + createFileHeaderLen)),
descriptionEvent);
((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen) : (headerLen + loadHeaderLen + createFileHeaderLen)),
descriptionEvent);
if (descriptionEvent.binlogVersion != 1)
{
fileId = buffer.getUint32(headerLen + loadHeaderLen
+ CF_FILE_ID_OFFSET);
if (descriptionEvent.binlogVersion != 1) {
fileId = buffer.getUint32(headerLen + loadHeaderLen + CF_FILE_ID_OFFSET);
/*
Note that it's ok to use get_data_size() below, because it is computed
with values we have already read from this event (because we called
copy_log_event()); we are not using slave's format info to decode
master's format, we are really using master's format info.
Anyway, both formats should be identical (except the common_header_len)
as these Load events are not changed between 4.0 and 5.0 (as logging of
LOAD DATA INFILE does not use Load_log_event in 5.0).
*/
* Note that it's ok to use get_data_size() below, because it is
* computed with values we have already read from this event
* (because we called copy_log_event()); we are not using slave's
* format info to decode master's format, we are really using
* master's format info. Anyway, both formats should be identical
* (except the common_header_len) as these Load events are not
* changed between 4.0 and 5.0 (as logging of LOAD DATA INFILE does
* not use Load_log_event in 5.0).
*/
blockLen = buffer.limit() - buffer.position();
blockBuf = buffer.duplicate(blockLen);
}
else
{
} else {
initedFromOld = true;
}
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
public final LogBuffer getBuffer()
{
public final LogBuffer getBuffer() {
return blockBuf;
}
public final byte[] getData()
{
public final byte[] getData() {
return blockBuf.getData();
}
}

13
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java
View File

@@ -9,25 +9,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class DeleteFileLogEvent extends LogEvent
{
public final class DeleteFileLogEvent extends LogEvent {
private final long fileId;
/* DF = "Delete File" */
public static final int DF_FILE_ID_OFFSET = 0;
public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET);
fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
}

8
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java
View File

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class DeleteRowsLogEvent extends RowsLogEvent
{
public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class DeleteRowsLogEvent extends RowsLogEvent {
public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

13
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java
View File

@@ -9,25 +9,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class ExecuteLoadLogEvent extends LogEvent
{
public final class ExecuteLoadLogEvent extends LogEvent {
private final long fileId;
/* EL = "Execute Load" */
public static final int EL_FILE_ID_OFFSET = 0;
public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET);
fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
}

35
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java
View File

@@ -8,7 +8,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* Event responsible for LOAD DATA execution, it similar to Query_log_event but
* before executing the query it substitutes original filename in LOAD DATA
* query with name of temporary file.
*
* <ul>
* <li>4 bytes. The ID of the file to load.</li>
* <li>4 bytes. The start position within the statement for filename
@@ -22,8 +21,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
{
public final class ExecuteLoadQueryLogEvent extends QueryLogEvent {
/** file_id of temporary file */
private long fileId;
@@ -53,9 +52,8 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
public static final int ELQ_FN_POS_END_OFFSET = ELQ_FILE_ID_OFFSET + 8;
public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent) throws IOException
{
public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
super(header, buffer, descriptionEvent);
buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
@@ -66,36 +64,27 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
final int len = query.length();
if (fnPosStart > len || fnPosEnd > len
|| dupHandling > LOAD_DUP_REPLACE)
{
throw new IOException(String.format(
"Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
+ "fn_pos_end=%d, dup_handling=%d", fnPosStart,
fnPosEnd, dupHandling));
if (fnPosStart > len || fnPosEnd > len || dupHandling > LOAD_DUP_REPLACE) {
throw new IOException(String.format("Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
+ "fn_pos_end=%d, dup_handling=%d", fnPosStart, fnPosEnd, dupHandling));
}
}
public final int getFilenamePosStart()
{
public final int getFilenamePosStart() {
return fnPosStart;
}
public final int getFilenamePosEnd()
{
public final int getFilenamePosEnd() {
return fnPosEnd;
}
public final String getFilename()
{
if (query != null)
return query.substring(fnPosStart, fnPosEnd).trim();
public final String getFilename() {
if (query != null) return query.substring(fnPosStart, fnPosEnd).trim();
return null;
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
}

147
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java
View File

@@ -5,8 +5,8 @@ import java.io.IOException;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
/**
* For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
* ordinary events).
* For binlog version 4. This event is saved by threads which read it, as they
* need it for future use (to decode the ordinary events).
*
* @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
@@ -15,23 +15,24 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
public final class FormatDescriptionLogEvent extends StartLogEventV3 {
/**
* The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
* exist in binlogs, it does not have a format).
* The number of types we handle in Format_description_log_event
* (UNKNOWN_EVENT is not to be handled, it does not exist in binlogs, it
* does not have a format).
*/
public static final int LOG_EVENT_TYPES = (ENUM_END_EVENT - 1);
public static final int LOG_EVENT_TYPES = (ENUM_END_EVENT - 1);
public static final int ST_COMMON_HEADER_LEN_OFFSET = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
public static final int ST_COMMON_HEADER_LEN_OFFSET = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
public static final int OLD_HEADER_LEN = 13;
public static final int LOG_EVENT_HEADER_LEN = 19;
public static final int LOG_EVENT_MINIMAL_HEADER_LEN = 19;
public static final int OLD_HEADER_LEN = 13;
public static final int LOG_EVENT_HEADER_LEN = 19;
public static final int LOG_EVENT_MINIMAL_HEADER_LEN = 19;
/* event-specific post-header sizes */
public static final int STOP_HEADER_LEN = 0;
public static final int LOAD_HEADER_LEN = (4 + 4 + 4 + 1 + 1 + 4);
public static final int SLAVE_HEADER_LEN = 0;
public static final int START_V3_HEADER_LEN = (2 + ST_SERVER_VER_LEN + 4);
public static final int ROTATE_HEADER_LEN = 8; // this
public static final int STOP_HEADER_LEN = 0;
public static final int LOAD_HEADER_LEN = (4 + 4 + 4 + 1 + 1 + 4);
public static final int SLAVE_HEADER_LEN = 0;
public static final int START_V3_HEADER_LEN = (2 + ST_SERVER_VER_LEN + 4);
public static final int ROTATE_HEADER_LEN = 8; // this
// is
// FROZEN
// (the
@@ -39,47 +40,48 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
// post-header
// is
// frozen)
public static final int INTVAR_HEADER_LEN = 0;
public static final int CREATE_FILE_HEADER_LEN = 4;
public static final int APPEND_BLOCK_HEADER_LEN = 4;
public static final int EXEC_LOAD_HEADER_LEN = 4;
public static final int DELETE_FILE_HEADER_LEN = 4;
public static final int NEW_LOAD_HEADER_LEN = LOAD_HEADER_LEN;
public static final int RAND_HEADER_LEN = 0;
public static final int USER_VAR_HEADER_LEN = 0;
public static final int FORMAT_DESCRIPTION_HEADER_LEN = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
public static final int XID_HEADER_LEN = 0;
public static final int BEGIN_LOAD_QUERY_HEADER_LEN = APPEND_BLOCK_HEADER_LEN;
public static final int ROWS_HEADER_LEN_V1 = 8;
public static final int TABLE_MAP_HEADER_LEN = 8;
public static final int EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
public static final int EXECUTE_LOAD_QUERY_HEADER_LEN = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
public static final int INCIDENT_HEADER_LEN = 2;
public static final int HEARTBEAT_HEADER_LEN = 0;
public static final int IGNORABLE_HEADER_LEN = 0;
public static final int ROWS_HEADER_LEN_V2 = 10;
public static final int ANNOTATE_ROWS_HEADER_LEN = 0;
public static final int BINLOG_CHECKPOINT_HEADER_LEN = 4;
public static final int GTID_HEADER_LEN = 19;
public static final int GTID_LIST_HEADER_LEN = 4;
public static final int INTVAR_HEADER_LEN = 0;
public static final int CREATE_FILE_HEADER_LEN = 4;
public static final int APPEND_BLOCK_HEADER_LEN = 4;
public static final int EXEC_LOAD_HEADER_LEN = 4;
public static final int DELETE_FILE_HEADER_LEN = 4;
public static final int NEW_LOAD_HEADER_LEN = LOAD_HEADER_LEN;
public static final int RAND_HEADER_LEN = 0;
public static final int USER_VAR_HEADER_LEN = 0;
public static final int FORMAT_DESCRIPTION_HEADER_LEN = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
public static final int XID_HEADER_LEN = 0;
public static final int BEGIN_LOAD_QUERY_HEADER_LEN = APPEND_BLOCK_HEADER_LEN;
public static final int ROWS_HEADER_LEN_V1 = 8;
public static final int TABLE_MAP_HEADER_LEN = 8;
public static final int EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
public static final int EXECUTE_LOAD_QUERY_HEADER_LEN = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
public static final int INCIDENT_HEADER_LEN = 2;
public static final int HEARTBEAT_HEADER_LEN = 0;
public static final int IGNORABLE_HEADER_LEN = 0;
public static final int ROWS_HEADER_LEN_V2 = 10;
public static final int ANNOTATE_ROWS_HEADER_LEN = 0;
public static final int BINLOG_CHECKPOINT_HEADER_LEN = 4;
public static final int GTID_HEADER_LEN = 19;
public static final int GTID_LIST_HEADER_LEN = 4;
public static final int POST_HEADER_LENGTH = 11;
public static final int POST_HEADER_LENGTH = 11;
public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
public static final int[] checksumVersionSplit = { 5, 6, 1 };
public static final long checksumVersionProduct = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
* 256 + checksumVersionSplit[2];
/**
* The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
* LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
* The size of the fixed header which _all_ events have (for binlogs written
* by this version, this is equal to LOG_EVENT_HEADER_LEN), except
* FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
* LOG_EVENT_MINIMAL_HEADER_LEN).
*/
protected final int commonHeaderLen;
protected int numberOfEventTypes;
protected final int commonHeaderLen;
protected int numberOfEventTypes;
/** The list of post-headers' lengthes */
protected final short[] postHeaderLen;
protected int[] serverVersionSplit = new int[3];
protected final short[] postHeaderLen;
protected int[] serverVersionSplit = new int[3];
public FormatDescriptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
@@ -105,12 +107,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
calcServerVersionSplit();
long calc = getVersionProduct();
if (calc >= checksumVersionProduct) {
/* the last bytes are the checksum alg desc and value (or value's room) */
/*
* the last bytes are the checksum alg desc and value (or value's
* room)
*/
numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
}
if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= "
+ numberOfEventTypes);
+ numberOfEventTypes);
}
/** MySQL 5.0 format descriptions. */
@@ -147,7 +152,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
commonHeaderLen = LOG_EVENT_HEADER_LEN;
numberOfEventTypes = LOG_EVENT_TYPES;
/* Note: all event types must explicitly fill in their lengths here. */
/*
* Note: all event types must explicitly fill in their lengths
* here.
*/
postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
@@ -171,11 +179,14 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
/*
* We here have the possibility to simulate a master of before we changed the table map id to be stored
* in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
* compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
* that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
* test (rpl_row_4_bytes) too.
* We here have the possibility to simulate a master of before
* we changed the table map id to be stored in 6 bytes: when it
* was stored in 4 bytes (=> post_header_len was 6). This is
* used to test backward compatibility. This code can be removed
* after a few months (today is Dec 21st 2005), when we know
* that the 4-byte masters are not deployed anymore (check with
* Tomas Ulin first!), and the accompanying test
* (rpl_row_4_bytes) too.
*/
postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
@@ -195,17 +206,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
case 3: /* 4.0.x x>=2 */
/*
* We build an artificial (i.e. not sent by the master) event, which describes what those old master
* versions send.
* We build an artificial (i.e. not sent by the master) event,
* which describes what those old master versions send.
*/
serverVersion = "4.0";
commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
/*
* The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
* in older versions. We use the events known by version 3, even if version 1 had only a subset of them
* (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
* detect less corruptions).
* The first new event in binlog version 4 is Format_desc. So
* any event type after that does not exist in older versions.
* We use the events known by version 3, even if version 1 had
* only a subset of them (this is not a problem: it uses a few
* bytes for nothing but unifies code; it does not make the
* slave detect less corruptions).
*/
numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
@@ -222,17 +235,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
case 1: /* 3.23 */
/*
* We build an artificial (i.e. not sent by the master) event, which describes what those old master
* versions send.
* We build an artificial (i.e. not sent by the master) event,
* which describes what those old master versions send.
*/
serverVersion = "3.23";
commonHeaderLen = OLD_HEADER_LEN;
/*
* The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
* in older versions. We use the events known by version 3, even if version 1 had only a subset of them
* (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
* detect less corruptions).
* The first new event in binlog version 4 is Format_desc. So
* any event type after that does not exist in older versions.
* We use the events known by version 3, even if version 1 had
* only a subset of them (this is not a problem: it uses a few
* bytes for nothing but unifies code; it does not make the
* slave detect less corruptions).
*/
numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
@@ -251,7 +266,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
commonHeaderLen = 0;
}
}
public void calcServerVersionSplit() {
doServerVersionSplit(serverVersion, serverVersionSplit);
}

26
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java
View File

@@ -6,19 +6,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* <pre>
* Base class for ignorable log events. Events deriving from
this class can be safely ignored by slaves that cannot
recognize them. Newer slaves, will be able to read and
handle them. This has been designed to be an open-ended
architecture, so adding new derived events shall not harm
the old slaves that support ignorable log event mechanism
(they will just ignore unrecognized ignorable events).
@note The only thing that makes an event ignorable is that it has
the LOG_EVENT_IGNORABLE_F flag set. It is not strictly necessary
that ignorable event types derive from Ignorable_log_event; they may
just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
argument to the Log_event constructor.
</pre>
* this class can be safely ignored by slaves that cannot
* recognize them. Newer slaves, will be able to read and
* handle them. This has been designed to be an open-ended
* architecture, so adding new derived events shall not harm
* the old slaves that support ignorable log event mechanism
* (they will just ignore unrecognized ignorable events).
*
* @note The only thing that makes an event ignorable is that it has
* the LOG_EVENT_IGNORABLE_F flag set. It is not strictly necessary
* that ignorable event types derive from Ignorable_log_event; they may
* just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
* argument to the Log_event constructor.
* </pre>
*
* @author jianghang 2013-4-8 上午12:36:29
* @version 1.0.3

28
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java
View File

@@ -5,12 +5,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* Class representing an incident, an occurance out of the ordinary, that
* happened on the master.
*
* The event is used to inform the slave that something out of the ordinary
* happened on the master that might cause the database to be in an inconsistent
* state.
*
* happened on the master. The event is used to inform the slave that something
* out of the ordinary happened on the master that might cause the database to
* be in an inconsistent state.
* <table id="IncidentFormat">
* <caption>Incident event format</caption>
* <tr>
@@ -38,8 +35,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class IncidentLogEvent extends LogEvent
{
public final class IncidentLogEvent extends LogEvent {
public static final int INCIDENT_NONE = 0;
/** There are possibly lost events in the replication stream */
@@ -51,9 +48,7 @@ public final class IncidentLogEvent extends LogEvent
private final int incident;
private final String message;
public IncidentLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public IncidentLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -61,10 +56,9 @@ public final class IncidentLogEvent extends LogEvent
buffer.position(commonHeaderLen);
final int incidentNumber = buffer.getUint16();
if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
{
if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE) {
// If the incident is not recognized, this binlog event is
// invalid. If we set incident_number to INCIDENT_NONE, the
// invalid. If we set incident_number to INCIDENT_NONE, the
// invalidity will be detected by is_valid().
incident = INCIDENT_NONE;
message = null;
@@ -76,13 +70,11 @@ public final class IncidentLogEvent extends LogEvent
message = buffer.getString();
}
public final int getIncident()
{
public final int getIncident() {
return incident;
}
public final String getMessage()
{
public final String getMessage() {
return message;
}
}

37
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java
View File

@@ -6,54 +6,43 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* An Intvar_log_event will be created just before a Query_log_event, if the
* query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
* Intvar_log_event holds the value of one of these variables.
*
* Binary Format
*
* The Post-Header for this event type is empty. The Body has two components:
*
* Intvar_log_event holds the value of one of these variables. Binary Format The
* Post-Header for this event type is empty. The Body has two components:
* <table>
* <caption>Body for Intvar_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>type</td>
* <td>1 byte enumeration</td>
* <td>One byte identifying the type of variable stored. Currently, two
* identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
* </tr>
*
* <tr>
* <td>value</td>
* <td>8 byte unsigned integer</td>
* <td>The value of the variable.</td>
* </tr>
*
* </table>
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class IntvarLogEvent extends LogEvent
{
public final class IntvarLogEvent extends LogEvent {
/**
* Fixed data part: Empty
*
* <p>
* Variable data part:
*
* <ul>
* <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
* 1 or INSERT_ID_EVENT = 2.</li>
* <li>8 bytes. An unsigned integer indicating the value to be used for the
* LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final long value;
@@ -68,31 +57,25 @@ public final class IntvarLogEvent extends LogEvent
public static final int LAST_INSERT_ID_EVENT = 1;
public static final int INSERT_ID_EVENT = 2;
public IntvarLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public IntvarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
/* The Post-Header is empty. The Varible Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+ I_TYPE_OFFSET);
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+ I_TYPE_OFFSET);
type = buffer.getInt8(); // I_TYPE_OFFSET
value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
}
public final int getType()
{
public final int getType() {
return type;
}
public final long getValue()
{
public final long getValue() {
return value;
}
public final String getQuery()
{
public final String getQuery() {
return "SET INSERT_ID = " + value;
}
}

139
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java
View File

@@ -26,16 +26,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* </pre>
*
* Binary Format: The Post-Header consists of the following six components.
*
* <table>
* <caption>Post-Header for Load_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>slave_proxy_id</td>
* <td>4 byte unsigned integer</td>
@@ -45,71 +42,57 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* temporary table local to the client. The slave_proxy_id is used to
* distinguish temporary tables that belong to different clients.</td>
* </tr>
*
* <tr>
* <td>exec_time</td>
* <td>4 byte unsigned integer</td>
* <td>The time from when the query started to when it was logged in the binlog,
* in seconds.</td>
* </tr>
*
* <tr>
* <td>skip_lines</td>
* <td>4 byte unsigned integer</td>
* <td>The number on line (14) above, if present, or 0 if line (14) is left out.
* </td>
* </tr>
*
* <tr>
* <td>table_name_len</td>
* <td>1 byte unsigned integer</td>
* <td>The length of 'table_name' on line (4) above.</td>
* </tr>
*
* <tr>
* <td>db_len</td>
* <td>1 byte unsigned integer</td>
* <td>The length of 'db' on line (1) above.</td>
* </tr>
*
* <tr>
* <td>num_fields</td>
* <td>4 byte unsigned integer</td>
* <td>The number n of fields on line (15) above.</td>
* </tr>
* </table>
*
* The Body contains the following components.
*
* <table>
* <caption>Body of Load_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>sql_ex</td>
* <td>variable length</td>
*
* <td>Describes the part of the query on lines (3) and (5)&ndash;(13) above.
* More precisely, it stores the five strings (on lines) field_term (6),
* enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
* bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
* OPTIONALLY (7).
*
* The data is stored in one of two formats, called "old" and "new". The type
* field of Common-Header determines which of these two formats is used: type
* LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
* that the new format is used. When MySQL writes a Load_log_event, it uses the
* new format if at least one of the five strings is two or more bytes long.
* Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
* used.
*
* The new and old format differ in the way the five strings are stored.
*
* OPTIONALLY (7). The data is stored in one of two formats, called "old" and
* "new". The type field of Common-Header determines which of these two formats
* is used: type LOAD_EVENT means that the old format is used, and type
* NEW_LOAD_EVENT means that the new format is used. When MySQL writes a
* Load_log_event, it uses the new format if at least one of the five strings is
* two or more bytes long. Otherwise (i.e., if all strings are 0 or 1 bytes
* long), the old format is used. The new and old format differ in the way the
* five strings are stored.
* <ul>
* <li>In the new format, the strings are stored in the order field_term,
* enclosed, escaped, line_term, line_start. Each string consists of a length (1
@@ -117,7 +100,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
* IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
* the presence of the corresponding keyword in the SQL query.
*
* <li>In the old format, we know that each string has length 0 or 1. Therefore,
* only the first byte of each string is stored. The order of the strings is the
* same as in the new format. These five bytes are followed by the same 1 byte
@@ -128,18 +110,15 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
* LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
* </ul>
*
* Thus, the size of the new format is 6 bytes + the sum of the sizes of the
* five strings. The size of the old format is always 7 bytes.</td>
* </tr>
*
* <tr>
* <td>field_lens</td>
* <td>num_fields 1 byte unsigned integers</td>
* <td>An array of num_fields integers representing the length of each field in
* the query. (num_fields is from the Post-Header).</td>
* </tr>
*
* <tr>
* <td>fields</td>
* <td>num_fields null-terminated strings</td>
@@ -148,7 +127,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* the num_fields array.) The total length of all strings equals to the sum of
* all field_lens, plus num_fields bytes for all the trailing zeros.</td>
* </tr>
*
* <tr>
* <td>table_name</td>
* <td>null-terminated string of length table_len+1 bytes</td>
@@ -156,31 +134,27 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* trailing zero is actually redundant since the table_len is known from
* Post-Header.)</td>
* </tr>
*
* <tr>
* <td>db</td>
* <td>null-terminated string of length db_len+1 bytes</td>
* <td>The 'db' from the query, as a null-terminated string. (The trailing zero
* is actually redundant since the db_len is known from Post-Header.)</td>
* </tr>
*
* <tr>
* <td>file_name</td>
* <td>variable length string without trailing zero, extending to the end of the
* event (determined by the length field of the Common-Header)</td>
* <td>The 'file_name' from the query.</td>
* </tr>
*
* </table>
*
* This event type is understood by current versions, but only generated by
* MySQL 3.23 and earlier.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class LoadLogEvent extends LogEvent
{
public class LoadLogEvent extends LogEvent {
private String table;
private String db;
private String fname;
@@ -210,11 +184,10 @@ public class LoadLogEvent extends LogEvent
public static final int L_DATA_OFFSET = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
/*
These are flags and structs to handle all the LOAD DATA INFILE options (LINES
TERMINATED etc).
DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
DATA).
*/
* These are flags and structs to handle all the LOAD DATA INFILE options
* (LINES TERMINATED etc). DUMPFILE_FLAG is probably useless (DUMPFILE is a
* clause of SELECT, not of LOAD DATA).
*/
public static final int DUMPFILE_FLAG = 0x1;
public static final int OPT_ENCLOSED_FLAG = 0x2;
public static final int REPLACE_FLAG = 0x4;
@@ -226,9 +199,7 @@ public class LoadLogEvent extends LogEvent
public static final int LINE_START_EMPTY = 0x8;
public static final int ESCAPED_EMPTY = 0x10;
public LoadLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public LoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
@@ -236,18 +207,16 @@ public class LoadLogEvent extends LogEvent
* I (Guilhem) manually tested replication of LOAD DATA INFILE for
* 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
*/
copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
+ descriptionEvent.commonHeaderLen : loadHeaderLen
+ FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
descriptionEvent);
copyLogEvent(buffer,
((header.type == LOAD_EVENT) ? loadHeaderLen + descriptionEvent.commonHeaderLen : loadHeaderLen
+ FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
descriptionEvent);
}
/**
* @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
*/
protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
FormatDescriptionLogEvent descriptionEvent)
{
protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset, FormatDescriptionLogEvent descriptionEvent) {
/* this is the beginning of the post-header */
buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
@@ -262,8 +231,7 @@ public class LoadLogEvent extends LogEvent
* Sql_ex.init() on success returns the pointer to the first byte after
* the sql_ex structure, which is the start of field lengths array.
*/
if (header.type != LOAD_EVENT /* use_new_format */)
{
if (header.type != LOAD_EVENT /* use_new_format */) {
/*
* The code below assumes that buf will not disappear from under our
* feet during the lifetime of the event. This assumption holds true
@@ -279,9 +247,7 @@ public class LoadLogEvent extends LogEvent
escaped = buffer.getString();
optFlags = buffer.getInt8();
emptyFlags = 0;
}
else
{
} else {
fieldTerm = buffer.getFixString(1);
enclosed = buffer.getFixString(1);
lineTerm = buffer.getFixString(1);
@@ -290,23 +256,17 @@ public class LoadLogEvent extends LogEvent
optFlags = buffer.getUint8();
emptyFlags = buffer.getUint8();
if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
fieldTerm = null;
if ((emptyFlags & ENCLOSED_EMPTY) != 0)
enclosed = null;
if ((emptyFlags & LINE_TERM_EMPTY) != 0)
lineTerm = null;
if ((emptyFlags & LINE_START_EMPTY) != 0)
lineStart = null;
if ((emptyFlags & ESCAPED_EMPTY) != 0)
escaped = null;
if ((emptyFlags & FIELD_TERM_EMPTY) != 0) fieldTerm = null;
if ((emptyFlags & ENCLOSED_EMPTY) != 0) enclosed = null;
if ((emptyFlags & LINE_TERM_EMPTY) != 0) lineTerm = null;
if ((emptyFlags & LINE_START_EMPTY) != 0) lineStart = null;
if ((emptyFlags & ESCAPED_EMPTY) != 0) escaped = null;
}
final int fieldLenPos = buffer.position();
buffer.forward(numFields);
fields = new String[numFields];
for (int i = 0; i < numFields; i++)
{
for (int i = 0; i < numFields; i++) {
final int fieldLen = buffer.getUint8(fieldLenPos + i);
fields[i] = buffer.getFixString(fieldLen + 1);
}
@@ -314,78 +274,65 @@ public class LoadLogEvent extends LogEvent
table = buffer.getFixString(tableNameLen + 1);
db = buffer.getFixString(dbLen + 1);
// null termination is accomplished by the caller
// null termination is accomplished by the caller
final int from = buffer.position();
final int end = from + buffer.limit();
int found = from;
for (; (found < end) && buffer.getInt8(found) != '\0'; found++)
/* empty loop */;
fname = buffer.getString(found);
buffer.forward(1); // The + 1 is for \0 terminating fname
buffer.forward(1); // The + 1 is for \0 terminating fname
}
public final String getTable()
{
public final String getTable() {
return table;
}
public final String getDb()
{
public final String getDb() {
return db;
}
public final String getFname()
{
public final String getFname() {
return fname;
}
public final int getSkipLines()
{
public final int getSkipLines() {
return skipLines;
}
public final String[] getFields()
{
public final String[] getFields() {
return fields;
}
public final String getFieldTerm()
{
public final String getFieldTerm() {
return fieldTerm;
}
public final String getLineTerm()
{
public final String getLineTerm() {
return lineTerm;
}
public final String getLineStart()
{
public final String getLineStart() {
return lineStart;
}
public final String getEnclosed()
{
public final String getEnclosed() {
return enclosed;
}
public final String getEscaped()
{
public final String getEscaped() {
return escaped;
}
public final int getOptFlags()
{
public final int getOptFlags() {
return optFlags;
}
public final int getEmptyFlags()
{
public final int getEmptyFlags() {
return emptyFlags;
}
public final long getExecTime()
{
public final long getExecTime() {
return execTime;
}
}

178
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java
View File

@@ -11,41 +11,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* different format and length even for different events of the same type. The
* binary formats of Post-Header and Body are documented separately in each
* subclass. The binary format of Common-Header is as follows.
*
* <table>
* <caption>Common-Header</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>timestamp</td>
* <td>4 byte unsigned integer</td>
* <td>The time when the query started, in seconds since 1970.</td>
* </tr>
*
* <tr>
* <td>type</td>
* <td>1 byte enumeration</td>
* <td>See enum #Log_event_type.</td>
* </tr>
*
* <tr>
* <td>server_id</td>
* <td>4 byte unsigned integer</td>
* <td>Server ID of the server that created the event.</td>
* </tr>
*
* <tr>
* <td>total_size</td>
* <td>4 byte unsigned integer</td>
* <td>The total size of this event, in bytes. In other words, this is the sum
* of the sizes of Common-Header, Post-Header, and Body.</td>
* </tr>
*
* <tr>
* <td>master_position</td>
* <td>4 byte unsigned integer</td>
@@ -54,24 +47,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* the position of the next event, in bytes from the beginning of the file. In a
* relay log, this is the position of the next event in the master's binlog.</td>
* </tr>
*
* <tr>
* <td>flags</td>
* <td>2 byte bitfield</td>
* <td>See Log_event::flags.</td>
* </tr>
* </table>
*
* Summing up the numbers above, we see that the total size of the common header
* is 19 bytes.
*
* @see mysql-5.1.60/sql/log_event.cc
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class LogHeader
{
public final class LogHeader {
protected final int type;
/**
@@ -111,37 +101,32 @@ public final class LogHeader
* LOG_EVENT_SUPPRESS_USE_F for notes.
*/
protected int flags;
/**
/**
* The value is set by caller of FD constructor and
* Log_event::write_header() for the rest.
* In the FD case it's propagated into the last byte
* of post_header_len[] at FD::write().
* On the slave side the value is assigned from post_header_len[last]
* of the last seen FD event.
* Log_event::write_header() for the rest. In the FD case it's propagated
* into the last byte of post_header_len[] at FD::write(). On the slave side
* the value is assigned from post_header_len[last] of the last seen FD
* event.
*/
protected int checksumAlg;
/**
Placeholder for event checksum while writing to binlog.
*/
protected long crc; // ha_checksum
* Placeholder for event checksum while writing to binlog.
*/
protected long crc; // ha_checksum
/* for Start_event_v3 */
public LogHeader(final int type)
{
public LogHeader(final int type){
this.type = type;
}
public LogHeader(LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public LogHeader(LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
when = buffer.getUint32();
type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET;
serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET;
eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET;
if (descriptionEvent.binlogVersion == 1)
{
if (descriptionEvent.binlogVersion == 1) {
logPos = 0;
flags = 0;
return;
@@ -150,50 +135,45 @@ public final class LogHeader
/* 4.0 or newer */
logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET
/*
If the log is 4.0 (so here it can only be a 4.0 relay log read by
the SQL thread or a 4.0 master binlog read by the I/O thread),
log_pos is the beginning of the event: we transform it into the end
of the event, which is more useful.
But how do you know that the log is 4.0: you know it if
description_event is version 3 *and* you are not reading a
Format_desc (remember that mysqlbinlog starts by assuming that 5.0
logs are in 4.0 format, until it finds a Format_desc).
*/
if (descriptionEvent.binlogVersion == 3
&& type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0)
{
* If the log is 4.0 (so here it can only be a 4.0 relay log read by the
* SQL thread or a 4.0 master binlog read by the I/O thread), log_pos is
* the beginning of the event: we transform it into the end of the
* event, which is more useful. But how do you know that the log is 4.0:
* you know it if description_event is version 3 *and* you are not
* reading a Format_desc (remember that mysqlbinlog starts by assuming
* that 5.0 logs are in 4.0 format, until it finds a Format_desc).
*/
if (descriptionEvent.binlogVersion == 3 && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0) {
/*
If log_pos=0, don't change it. log_pos==0 is a marker to mean
"don't change rli->group_master_log_pos" (see
inc_group_relay_log_pos()). As it is unreal log_pos, adding the
event len's is nonsense. For example, a fake Rotate event should
not have its log_pos (which is 0) changed or it will modify
Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
value of (a non-zero offset which does not exist in the master's
binlog, so which will cause problems if the user uses this value
in CHANGE MASTER).
*/
* If log_pos=0, don't change it. log_pos==0 is a marker to mean
* "don't change rli->group_master_log_pos" (see
* inc_group_relay_log_pos()). As it is unreal log_pos, adding the
* event len's is nonsense. For example, a fake Rotate event should
* not have its log_pos (which is 0) changed or it will modify
* Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
* value of (a non-zero offset which does not exist in the master's
* binlog, so which will cause problems if the user uses this value
* in CHANGE MASTER).
*/
logPos += eventLen; /* purecov: inspected */
}
flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET
if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT)
|| (type == LogEvent.ROTATE_EVENT))
{
if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT) || (type == LogEvent.ROTATE_EVENT)) {
/*
These events always have a header which stops here (i.e. their
header is FROZEN).
*/
* These events always have a header which stops here (i.e. their
* header is FROZEN).
*/
/*
Initialization to zero of all other Log_event members as they're
not specified. Currently there are no such members; in the future
there will be an event UID (but Format_description and Rotate
don't need this UID, as they are not propagated through
--log-slave-updates (remember the UID is used to not play a query
twice when you have two masters which are slaves of a 3rd master).
Then we are done.
*/
* Initialization to zero of all other Log_event members as they're
* not specified. Currently there are no such members; in the future
* there will be an event UID (but Format_description and Rotate
* don't need this UID, as they are not propagated through
* --log-slave-updates (remember the UID is used to not play a query
* twice when you have two masters which are slaves of a 3rd
* master). Then we are done.
*/
if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) {
int commonHeaderLen = buffer.getUint8(FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN
+ FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET);
@@ -211,27 +191,24 @@ public final class LogHeader
}
return;
}
/*
CRC verification by SQL and Show-Binlog-Events master side.
The caller has to provide @description_event->checksum_alg to
be the last seen FD's (A) descriptor.
If event is FD the descriptor is in it.
Notice, FD of the binlog can be only in one instance and therefore
Show-Binlog-Events executing master side thread needs just to know
the only FD's (A) value - whereas RL can contain more.
In the RL case, the alg is kept in FD_e (@description_event) which is reset
to the newer read-out event after its execution with possibly new alg descriptor.
Therefore in a typical sequence of RL:
{FD_s^0, FD_m, E_m^1} E_m^1
will be verified with (A) of FD_m.
See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs
lines (log.h).
Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF.
*/
checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg
* CRC verification by SQL and Show-Binlog-Events master side. The
* caller has to provide @description_event->checksum_alg to be the last
* seen FD's (A) descriptor. If event is FD the descriptor is in it.
* Notice, FD of the binlog can be only in one instance and therefore
* Show-Binlog-Events executing master side thread needs just to know
* the only FD's (A) value - whereas RL can contain more. In the RL
* case, the alg is kept in FD_e (@description_event) which is reset to
* the newer read-out event after its execution with possibly new alg
* descriptor. Therefore in a typical sequence of RL: {FD_s^0, FD_m,
* E_m^1} E_m^1 will be verified with (A) of FD_m. See legends
* definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs lines
* (log.h). Notice, a pre-checksum FD version forces alg :=
* BINLOG_CHECKSUM_ALG_UNDEF.
*/
checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch
// checksum alg
processCheckSum(buffer);
/* otherwise, go on with reading the header from buf (nothing now) */
}
@@ -239,8 +216,7 @@ public final class LogHeader
/**
* The different types of log events.
*/
public final int getType()
{
public final int getType() {
return type;
}
@@ -251,8 +227,7 @@ public final class LogHeader
* file. In a relay log, this is the position of the next event in the
* master's binlog.
*/
public final long getLogPos()
{
public final long getLogPos() {
return logPos;
}
@@ -260,24 +235,21 @@ public final class LogHeader
* The total size of this event, in bytes. In other words, this is the sum
* of the sizes of Common-Header, Post-Header, and Body.
*/
public final int getEventLen()
{
public final int getEventLen() {
return eventLen;
}
/**
* The time when the query started, in seconds since 1970.
*/
public final long getWhen()
{
public final long getWhen() {
return when;
}
/**
* Server ID of the server that created the event.
*/
public final long getServerId()
{
public final long getServerId() {
return serverId;
}
@@ -286,25 +258,21 @@ public final class LogHeader
* LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
* LOG_EVENT_SUPPRESS_USE_F for notes.
*/
public final int getFlags()
{
public final int getFlags() {
return flags;
}
public long getCrc() {
return crc;
}
public int getChecksumAlg() {
return checksumAlg;
}
private void processCheckSum(LogBuffer buffer) {
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF &&
checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){
crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
}
}
}

4
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java
View File

@@ -4,16 +4,14 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
*
* @author jianghang 2013-4-8 上午12:36:29
* @version 1.0.3
* @since mysql 5.6
*/
public class PreviousGtidsLogEvent extends LogEvent{
public class PreviousGtidsLogEvent extends LogEvent {
public PreviousGtidsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
// do nothing , just for mysql gtid search function
}
}

623
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
View File

@@ -9,19 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* A Query_log_event is created for each query that modifies the database,
* unless the query is logged row-based.
*
* The Post-Header has five components:
*
* unless the query is logged row-based. The Post-Header has five components:
* <table>
* <caption>Post-Header for Query_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>slave_proxy_id</td>
* <td>4 byte unsigned integer</td>
@@ -31,20 +26,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* temporary table local to the client. The slave_proxy_id is used to
* distinguish temporary tables that belong to different clients.</td>
* </tr>
*
* <tr>
* <td>exec_time</td>
* <td>4 byte unsigned integer</td>
* <td>The time from when the query started to when it was logged in the binlog,
* in seconds.</td>
* </tr>
*
* <tr>
* <td>db_len</td>
* <td>1 byte integer</td>
* <td>The length of the name of the currently selected database.</td>
* </tr>
*
* <tr>
* <td>error_code</td>
* <td>2 byte unsigned integer</td>
@@ -52,7 +44,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* fail with the same error code, except for the error codes ER_DB_CREATE_EXISTS
* == 1007 and ER_DB_DROP_EXISTS == 1008.</td>
* </tr>
*
* <tr>
* <td>status_vars_len</td>
* <td>2 byte unsigned integer</td>
@@ -60,18 +51,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* query_log_event_status_vars "below".</td>
* </tr>
* </table>
*
* The Body has the following components:
*
* <table>
* <caption>Body for Query_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>query_log_event_status_vars status_vars</td>
* <td>status_vars_len bytes</td>
@@ -81,16 +68,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* Table_query_log_event_status_vars "below". MySQL always writes events in the
* order defined below; however, it is capable of reading them in any order.</td>
* </tr>
*
* <tr>
* <td>db</td>
* <td>db_len+1</td>
* <td>The currently selected database, as a null-terminated string.
*
* (The trailing zero is redundant since the length is already known; it is
* db_len from Post-Header.)</td>
* <td>The currently selected database, as a null-terminated string. (The
* trailing zero is redundant since the length is already known; it is db_len
* from Post-Header.)</td>
* </tr>
*
* <tr>
* <td>query</td>
* <td>variable length string without trailing zero, extending to the end of the
@@ -98,20 +82,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>The SQL query.</td>
* </tr>
* </table>
*
* The following table lists the status variables that may appear in the
* status_vars field. Table_query_log_event_status_vars
*
* <table>
* <caption>Status variables for Query_log_event</caption>
*
* <tr>
* <th>Status variable</th>
* <th>1 byte identifier</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>flags2</td>
* <td>Q_FLAGS2_CODE == 0</td>
@@ -121,16 +101,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* identifies those options that need to be written to the binlog (not all do).
* Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL |
* OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
* OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex.
*
* These flags correspond to the SQL variables SQL_AUTO_IS_NULL,
* FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and AUTOCOMMIT, documented in the
* "SET Syntax" section of the MySQL Manual.
*
* This field is always written to the binlog in version >= 5.0, and never
* written in version < 5.0.</td>
* OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. These flags correspond to the
* SQL variables SQL_AUTO_IS_NULL, FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and
* AUTOCOMMIT, documented in the "SET Syntax" section of the MySQL Manual. This
* field is always written to the binlog in version >= 5.0, and never written in
* version < 5.0.</td>
* </tr>
*
* <tr>
* <td>sql_mode</td>
* <td>Q_SQL_MODE_CODE == 1</td>
@@ -177,11 +153,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* All these flags are replicated from the server. However, all flags except
* MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always preserves
* its old value of MODE_NO_DIR_IN_CREATE. For a rationale, see comment in
* Query_log_event::do_apply_event in log_event.cc.
*
* This field is always written to the binlog.</td>
* Query_log_event::do_apply_event in log_event.cc. This field is always written
* to the binlog.</td>
* </tr>
*
* <tr>
* <td>catalog</td>
* <td>Q_CATALOG_NZ_CODE == 6</td>
@@ -189,23 +163,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* characters (at most 255 bytes)</td>
* <td>Stores the client's current catalog. Every database belongs to a catalog,
* the same way that every table belongs to a database. Currently, there is only
* one catalog, "std".
*
* This field is written if the length of the catalog is > 0; otherwise it is
* not written.</td>
* one catalog, "std". This field is written if the length of the catalog is >
* 0; otherwise it is not written.</td>
* </tr>
*
* <tr>
* <td>auto_increment</td>
* <td>Q_AUTO_INCREMENT == 3</td>
* <td>two 2 byte unsigned integers, totally 2+2=4 bytes</td>
*
* <td>The two variables auto_increment_increment and auto_increment_offset, in
* that order. For more information, see "System variables" in the MySQL manual.
*
* This field is written if auto_increment > 1. Otherwise, it is not written.</td>
* </tr>
*
* <tr>
* <td>charset</td>
* <td>Q_CHARSET_CODE == 4</td>
@@ -216,33 +184,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* collation_connection identifies the character set and collation that the
* master converts the query to when it receives it; this is useful when
* comparing literal strings. collation_server is the default character set and
* collation used when a new database is created.
*
* See also "Connection Character Sets and Collations" in the MySQL 5.1 manual.
*
* All three variables are codes identifying a (character set, collation) pair.
* To see which codes map to which pairs, run the query "SELECT id,
* character_set_name, collation_name FROM COLLATIONS".
*
* Cf. Q_CHARSET_DATABASE_CODE below.
*
* This field is always written.</td>
* collation used when a new database is created. See also
* "Connection Character Sets and Collations" in the MySQL 5.1 manual. All three
* variables are codes identifying a (character set, collation) pair. To see
* which codes map to which pairs, run the query "SELECT id, character_set_name,
* collation_name FROM COLLATIONS". Cf. Q_CHARSET_DATABASE_CODE below. This
* field is always written.</td>
* </tr>
*
* <tr>
* <td>time_zone</td>
* <td>Q_TIME_ZONE_CODE == 5</td>
* <td>Variable-length string: the length in bytes (1 byte) followed by the
* characters (at most 255 bytes).
* <td>The time_zone of the master.
*
* See also "System Variables" and "MySQL Server Time Zone Support" in the MySQL
* manual.
*
* This field is written if the length of the time zone string is > 0;
* otherwise, it is not written.</td>
* <td>The time_zone of the master. See also "System Variables" and
* "MySQL Server Time Zone Support" in the MySQL manual. This field is written
* if the length of the time zone string is > 0; otherwise, it is not written.</td>
* </tr>
*
* <tr>
* <td>lc_time_names_number</td>
* <td>Q_LC_TIME_NAMES_CODE == 7</td>
@@ -251,84 +208,68 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* to languages is defined in sql_locale.cc. This field is written if it is not
* 0, i.e., if the locale is not en_US.</td>
* </tr>
*
* <tr>
* <td>charset_database_number</td>
* <td>Q_CHARSET_DATABASE_CODE == 8</td>
* <td>2 byte integer</td>
*
* <td>The value of the collation_database system variable (in the source code
* stored in thd->variables.collation_database), which holds the code for a
* (character set, collation) pair as described above (see Q_CHARSET_CODE).
*
* collation_database was used in old versions (???WHEN). Its value was loaded
* when issuing a "use db" query and could be changed by issuing a
* "SET collation_database=xxx" query. It used to affect the "LOAD DATA INFILE"
* and "CREATE TABLE" commands.
*
* In newer versions, "CREATE TABLE" has been changed to take the character set
* from the database of the created table, rather than the character set of the
* current database. This makes a difference when creating a table in another
* database than the current one. "LOAD DATA INFILE" has not yet changed to do
* this, but there are plans to eventually do it, and to make collation_database
* read-only.
*
* This field is written if it is not 0.</td>
* and "CREATE TABLE" commands. In newer versions, "CREATE TABLE" has been
* changed to take the character set from the database of the created table,
* rather than the character set of the current database. This makes a
* difference when creating a table in another database than the current one.
* "LOAD DATA INFILE" has not yet changed to do this, but there are plans to
* eventually do it, and to make collation_database read-only. This field is
* written if it is not 0.</td>
* </tr>
* <tr>
* <td>table_map_for_update</td>
* <td>Q_TABLE_MAP_FOR_UPDATE_CODE == 9</td>
* <td>8 byte integer</td>
*
* <td>The value of the table map that is to be updated by the multi-table
* update query statement. Every bit of this variable represents a table, and is
* set to 1 if the corresponding table is to be updated by this statement.
*
* The value of this variable is set when executing a multi-table update
* statement and used by slave to apply filter rules without opening all the
* tables on slave. This is required because some tables may not exist on slave
* because of the filter rules.</td>
* set to 1 if the corresponding table is to be updated by this statement. The
* value of this variable is set when executing a multi-table update statement
* and used by slave to apply filter rules without opening all the tables on
* slave. This is required because some tables may not exist on slave because of
* the filter rules.</td>
* </tr>
* </table>
*
* Query_log_event_notes_on_previous_versions Notes on Previous Versions
*
* Status vars were introduced in version 5.0. To read earlier versions
* correctly, check the length of the Post-Header.
*
* The status variable Q_CATALOG_CODE == 2 existed in MySQL 5.0.x, where
* 0<=x<=3. It was identical to Q_CATALOG_CODE, except that the string had a
* trailing '\0'. The '\0' was removed in 5.0.4 since it was redundant (the
* string length is stored before the string). The Q_CATALOG_CODE will never be
* written by a new master, but can still be understood by a new slave.
*
* See Q_CHARSET_DATABASE_CODE in the table above.
*
* Query_log_event_notes_on_previous_versions Notes on Previous Versions Status
* vars were introduced in version 5.0. To read earlier versions correctly,
* check the length of the Post-Header. The status variable Q_CATALOG_CODE == 2
* existed in MySQL 5.0.x, where 0<=x<=3. It was identical to Q_CATALOG_CODE,
* except that the string had a trailing '\0'. The '\0' was removed in 5.0.4
* since it was redundant (the string length is stored before the string). The
* Q_CATALOG_CODE will never be written by a new master, but can still be
* understood by a new slave. See Q_CHARSET_DATABASE_CODE in the table above.
* When adding new status vars, please don't forget to update the
* MAX_SIZE_LOG_EVENT_STATUS, and update function code_name
*
* @see mysql-5.1.6/sql/logevent.cc - Query_log_event
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class QueryLogEvent extends LogEvent
{
public class QueryLogEvent extends LogEvent {
/**
The maximum number of updated databases that a status of
Query-log-event can carry. It can redefined within a range
[1.. OVER_MAX_DBS_IN_EVENT_MTS].
* The maximum number of updated databases that a status of Query-log-event
* can carry. It can redefined within a range [1..
* OVER_MAX_DBS_IN_EVENT_MTS].
*/
public static final int MAX_DBS_IN_EVENT_MTS = 16;
/**
When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS
the value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the
mts_accessed_dbs status.
* When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS the
* value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the mts_accessed_dbs
* status.
*/
public static final int OVER_MAX_DBS_IN_EVENT_MTS = 254;
public static final int SYSTEM_CHARSET_MBMAXLEN = 3;
public static final int NAME_CHAR_LEN = 64;
/* Field/table name length */
@@ -340,23 +281,68 @@ public class QueryLogEvent extends LogEvent
* log_event status vars estimation:
*/
public static final int MAX_SIZE_LOG_EVENT_STATUS = (1 + 4 /* type, flags2 */
+ 1 + 8 /* type, sql_mode */
+ 1 + 1 + 255 /* type, length, catalog */
+ 1 + 4 /* type, auto_increment */
+ 1 + 6 /* type, charset */
+ 1 + 1 + 255 /* type, length, time_zone */
+ 1 + 2 /* type, lc_time_names_number */
+ 1 + 2 /* type, charset_database_number */
+ 1 + 8 /* type, table_map_for_update */
+ 1 + 4 /* type, master_data_written */
/* type, db_1, db_2, ... */
/* type, microseconds */
/* MariaDb type, sec_part of NOW() */
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3
+ 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */);
+ 1 + 8 /*
* type,
* sql_mode
*/
+ 1 + 1 + 255 /*
* type,
* length
* ,
* catalog
*/
+ 1 + 4 /*
* type,
* auto_increment
*/
+ 1 + 6 /*
* type,
* charset
*/
+ 1 + 1 + 255 /*
* type,
* length
* ,
* time_zone
*/
+ 1 + 2 /*
* type,
* lc_time_names_number
*/
+ 1 + 2 /*
* type,
* charset_database_number
*/
+ 1 + 8 /*
* type,
* table_map_for_update
*/
+ 1 + 4 /*
* type,
* master_data_written
*/
/*
* type, db_1, db_2,
* ...
*/
/* type, microseconds */
/*
* MariaDb type,
* sec_part of NOW()
*/
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 16 + 1 + 60/*
* type
* ,
* user_len
* ,
* user
* ,
* host_len
* ,
* host
*/);
/**
* Fixed data part:
*
* <ul>
* <li>4 bytes. The ID of the thread that issued this statement. Needed for
* temporary tables. This is also useful for a DBA for knowing who did what
@@ -385,7 +371,6 @@ public class QueryLogEvent extends LogEvent
* <li>2 bytes (not present in v1, v3). The length of the status variable
* block.</li>
* </ul>
*
* Variable part:
* <ul>
* <li>Zero or more status variables (not present in v1, v3). Each status
@@ -397,7 +382,6 @@ public class QueryLogEvent extends LogEvent
* the variable part (the sizes are given in the fixed data part), so by
* subtraction it can know the size of the statement.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private String user;
@@ -409,10 +393,11 @@ public class QueryLogEvent extends LogEvent
protected final String dbname;
/** The number of seconds the query took to run on the master. */
// The time in seconds that the statement took to execute. Only useful for inspection by the DBA
// The time in seconds that the statement took to execute. Only useful for
// inspection by the DBA
private final long execTime;
private final int errorCode;
private final long sessionId; /* thread_id */
private final long sessionId; /* thread_id */
/**
* 'flags2' is a second set of flags (on top of those in Log_event), for
@@ -434,9 +419,8 @@ public class QueryLogEvent extends LogEvent
private String timezone;
public QueryLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent) throws IOException
{
public QueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -447,8 +431,7 @@ public class QueryLogEvent extends LogEvent
* format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same
* for 3.23, 4.0 & 5.0.
*/
if (buffer.limit() < (commonHeaderLen + postHeaderLen))
{
if (buffer.limit() < (commonHeaderLen + postHeaderLen)) {
throw new IOException("Query event length is too short.");
}
int dataLen = buffer.limit() - (commonHeaderLen + postHeaderLen);
@@ -462,24 +445,21 @@ public class QueryLogEvent extends LogEvent
errorCode = buffer.getUint16(); // Q_ERR_CODE_OFFSET
/*
* 5.0 format starts here. Depending on the format, we may or not
* have affected/warnings etc The remaining post-header to be parsed
* has length:
* 5.0 format starts here. Depending on the format, we may or not have
* affected/warnings etc The remaining post-header to be parsed has
* length:
*/
int statusVarsLen = 0;
if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN)
{
if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN) {
statusVarsLen = buffer.getUint16(); // Q_STATUS_VARS_LEN_OFFSET
/*
Check if status variable length is corrupt and will lead to very
wrong data. We could be even more strict and require data_len to
be even bigger, but this will suffice to catch most corruption
errors that can lead to a crash.
*/
if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS))
{
throw new IOException("status_vars_len (" + statusVarsLen
+ ") > data_len (" + dataLen + ")");
* Check if status variable length is corrupt and will lead to very
* wrong data. We could be even more strict and require data_len to
* be even bigger, but this will suffice to catch most corruption
* errors that can lead to a crash.
*/
if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS)) {
throw new IOException("status_vars_len (" + statusVarsLen + ") > data_len (" + dataLen + ")");
}
dataLen -= statusVarsLen;
}
@@ -489,7 +469,7 @@ public class QueryLogEvent extends LogEvent
* dedicated to derived events (e.g. Execute_load_query...)
*/
/* variable-part: the status vars; only in MySQL 5.0 */
/* variable-part: the status vars; only in MySQL 5.0 */
final int start = commonHeaderLen + postHeaderLen;
final int limit = buffer.limit(); /* for restore */
final int end = start + statusVarsLen;
@@ -501,27 +481,19 @@ public class QueryLogEvent extends LogEvent
/* A 2nd variable part; this is common to all versions */
final int queryLen = dataLen - dbLen - 1;
dbname = buffer.getFixString(dbLen + 1);
if (clientCharset >= 0)
{
if (clientCharset >= 0) {
charsetName = CharsetConversion.getJavaCharset(clientCharset);
if ((charsetName != null) && (Charset.isSupported(charsetName)))
{
if ((charsetName != null) && (Charset.isSupported(charsetName))) {
query = buffer.getFixString(queryLen, charsetName);
}
else
{
logger.warn("unsupported character set in query log: "
+ "\n ID = " + clientCharset + ", Charset = "
+ CharsetConversion.getCharset(clientCharset)
+ ", Collation = "
+ CharsetConversion.getCollation(clientCharset));
} else {
logger.warn("unsupported character set in query log: " + "\n ID = " + clientCharset + ", Charset = "
+ CharsetConversion.getCharset(clientCharset) + ", Collation = "
+ CharsetConversion.getCollation(clientCharset));
query = buffer.getFixString(queryLen);
}
}
else
{
} else {
query = buffer.getFixString(queryLen);
}
}
@@ -567,215 +539,198 @@ public class QueryLogEvent extends LogEvent
public static final int Q_MASTER_DATA_WRITTEN_CODE = 10;
public static final int Q_INVOKER = 11;
/**
Q_UPDATED_DB_NAMES status variable collects of the updated databases
total number and their names to be propagated to the slave in order
to facilitate the parallel applying of the Query events.
* Q_UPDATED_DB_NAMES status variable collects of the updated databases
* total number and their names to be propagated to the slave in order to
* facilitate the parallel applying of the Query events.
*/
public static final int Q_UPDATED_DB_NAMES = 12;
public static final int Q_MICROSECONDS = 13;
/**
* FROM MariaDB 5.5.34
/**
* FROM MariaDB 5.5.34
*/
public static final int Q_HRNOW = 128;
private final void unpackVariables(LogBuffer buffer, final int end)
throws IOException
{
private final void unpackVariables(LogBuffer buffer, final int end) throws IOException {
int code = -1;
try
{
while (buffer.position() < end)
{
switch (code = buffer.getUint8())
{
case Q_FLAGS2_CODE:
flags2 = buffer.getUint32();
break;
case Q_SQL_MODE_CODE:
sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode is ulonglong
break;
case Q_CATALOG_NZ_CODE:
catalog = buffer.getString();
break;
case Q_AUTO_INCREMENT:
autoIncrementIncrement = buffer.getUint16();
autoIncrementOffset = buffer.getUint16();
break;
case Q_CHARSET_CODE:
// Charset: 6 byte character set flag.
// 1-2 = character set client
// 3-4 = collation client
// 5-6 = collation server
clientCharset = buffer.getUint16();
clientCollation = buffer.getUint16();
serverCollation = buffer.getUint16();
break;
case Q_TIME_ZONE_CODE:
timezone = buffer.getString();
break;
case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
final int len = buffer.getUint8();
catalog = buffer.getFixString(len + 1);
break;
case Q_LC_TIME_NAMES_CODE:
// lc_time_names_number = buffer.getUint16();
buffer.forward(2);
break;
case Q_CHARSET_DATABASE_CODE:
// charset_database_number = buffer.getUint16();
buffer.forward(2);
break;
case Q_TABLE_MAP_FOR_UPDATE_CODE:
// table_map_for_update = buffer.getUlong64();
buffer.forward(8);
break;
case Q_MASTER_DATA_WRITTEN_CODE:
// data_written = master_data_written = buffer.getUint32();
buffer.forward(4);
break;
case Q_INVOKER:
user = buffer.getString();
host = buffer.getString();
break;
case Q_MICROSECONDS:
// when.tv_usec= uint3korr(pos);
buffer.forward(3);
break;
case Q_UPDATED_DB_NAMES:
int mtsAccessedDbs = buffer.getUint8();
/**
Notice, the following check is positive also in case of
the master's MAX_DBS_IN_EVENT_MTS > the slave's one and the event
contains e.g the master's MAX_DBS_IN_EVENT_MTS db:s.
*/
if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) {
mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS;
try {
while (buffer.position() < end) {
switch (code = buffer.getUint8()) {
case Q_FLAGS2_CODE:
flags2 = buffer.getUint32();
break;
}
String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
int length = end - buffer.position();
mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
}
break;
case Q_HRNOW:
// int when_sec_part = buffer.getUint24();
buffer.forward(3);
break;
default:
/* That's why you must write status vars in growing order of code */
if (logger.isDebugEnabled())
logger.debug("Query_log_event has unknown status vars (first has code: "
+ code + "), skipping the rest of them");
break; // Break loop
case Q_SQL_MODE_CODE:
sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode
// is ulonglong
break;
case Q_CATALOG_NZ_CODE:
catalog = buffer.getString();
break;
case Q_AUTO_INCREMENT:
autoIncrementIncrement = buffer.getUint16();
autoIncrementOffset = buffer.getUint16();
break;
case Q_CHARSET_CODE:
// Charset: 6 byte character set flag.
// 1-2 = character set client
// 3-4 = collation client
// 5-6 = collation server
clientCharset = buffer.getUint16();
clientCollation = buffer.getUint16();
serverCollation = buffer.getUint16();
break;
case Q_TIME_ZONE_CODE:
timezone = buffer.getString();
break;
case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
final int len = buffer.getUint8();
catalog = buffer.getFixString(len + 1);
break;
case Q_LC_TIME_NAMES_CODE:
// lc_time_names_number = buffer.getUint16();
buffer.forward(2);
break;
case Q_CHARSET_DATABASE_CODE:
// charset_database_number = buffer.getUint16();
buffer.forward(2);
break;
case Q_TABLE_MAP_FOR_UPDATE_CODE:
// table_map_for_update = buffer.getUlong64();
buffer.forward(8);
break;
case Q_MASTER_DATA_WRITTEN_CODE:
// data_written = master_data_written =
// buffer.getUint32();
buffer.forward(4);
break;
case Q_INVOKER:
user = buffer.getString();
host = buffer.getString();
break;
case Q_MICROSECONDS:
// when.tv_usec= uint3korr(pos);
buffer.forward(3);
break;
case Q_UPDATED_DB_NAMES:
int mtsAccessedDbs = buffer.getUint8();
/**
* Notice, the following check is positive also in case
* of the master's MAX_DBS_IN_EVENT_MTS > the slave's
* one and the event contains e.g the master's
* MAX_DBS_IN_EVENT_MTS db:s.
*/
if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) {
mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS;
break;
}
String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
int length = end - buffer.position();
mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
}
break;
case Q_HRNOW:
// int when_sec_part = buffer.getUint24();
buffer.forward(3);
break;
default:
/*
* That's why you must write status vars in growing
* order of code
*/
if (logger.isDebugEnabled()) logger.debug("Query_log_event has unknown status vars (first has code: "
+ code + "), skipping the rest of them");
break; // Break loop
}
}
}
catch (RuntimeException e)
{
throw new IOException("Read " + findCodeName(code) + " error: "
+ e.getMessage(), e);
} catch (RuntimeException e) {
throw new IOException("Read " + findCodeName(code) + " error: " + e.getMessage(), e);
}
}
private static final String findCodeName(final int code)
{
switch (code)
{
case Q_FLAGS2_CODE:
return "Q_FLAGS2_CODE";
case Q_SQL_MODE_CODE:
return "Q_SQL_MODE_CODE";
case Q_CATALOG_CODE:
return "Q_CATALOG_CODE";
case Q_AUTO_INCREMENT:
return "Q_AUTO_INCREMENT";
case Q_CHARSET_CODE:
return "Q_CHARSET_CODE";
case Q_TIME_ZONE_CODE:
return "Q_TIME_ZONE_CODE";
case Q_CATALOG_NZ_CODE:
return "Q_CATALOG_NZ_CODE";
case Q_LC_TIME_NAMES_CODE:
return "Q_LC_TIME_NAMES_CODE";
case Q_CHARSET_DATABASE_CODE:
return "Q_CHARSET_DATABASE_CODE";
case Q_TABLE_MAP_FOR_UPDATE_CODE:
return "Q_TABLE_MAP_FOR_UPDATE_CODE";
case Q_MASTER_DATA_WRITTEN_CODE:
return "Q_MASTER_DATA_WRITTEN_CODE";
case Q_UPDATED_DB_NAMES:
return "Q_UPDATED_DB_NAMES";
case Q_MICROSECONDS:
return "Q_MICROSECONDS";
private static final String findCodeName(final int code) {
switch (code) {
case Q_FLAGS2_CODE:
return "Q_FLAGS2_CODE";
case Q_SQL_MODE_CODE:
return "Q_SQL_MODE_CODE";
case Q_CATALOG_CODE:
return "Q_CATALOG_CODE";
case Q_AUTO_INCREMENT:
return "Q_AUTO_INCREMENT";
case Q_CHARSET_CODE:
return "Q_CHARSET_CODE";
case Q_TIME_ZONE_CODE:
return "Q_TIME_ZONE_CODE";
case Q_CATALOG_NZ_CODE:
return "Q_CATALOG_NZ_CODE";
case Q_LC_TIME_NAMES_CODE:
return "Q_LC_TIME_NAMES_CODE";
case Q_CHARSET_DATABASE_CODE:
return "Q_CHARSET_DATABASE_CODE";
case Q_TABLE_MAP_FOR_UPDATE_CODE:
return "Q_TABLE_MAP_FOR_UPDATE_CODE";
case Q_MASTER_DATA_WRITTEN_CODE:
return "Q_MASTER_DATA_WRITTEN_CODE";
case Q_UPDATED_DB_NAMES:
return "Q_UPDATED_DB_NAMES";
case Q_MICROSECONDS:
return "Q_MICROSECONDS";
}
return "CODE#" + code;
}
public final String getUser()
{
public final String getUser() {
return user;
}
public final String getHost()
{
public final String getHost() {
return host;
}
public final String getQuery()
{
public final String getQuery() {
return query;
}
public final String getCatalog()
{
public final String getCatalog() {
return catalog;
}
public final String getDbName()
{
public final String getDbName() {
return dbname;
}
/**
* The number of seconds the query took to run on the master.
*/
public final long getExecTime()
{
public final long getExecTime() {
return execTime;
}
public final int getErrorCode()
{
public final int getErrorCode() {
return errorCode;
}
public final long getSessionId()
{
public final long getSessionId() {
return sessionId;
}
public final long getAutoIncrementIncrement()
{
public final long getAutoIncrementIncrement() {
return autoIncrementIncrement;
}
public final long getAutoIncrementOffset()
{
public final long getAutoIncrementOffset() {
return autoIncrementOffset;
}
public final String getCharsetName()
{
public final String getCharsetName() {
return charsetName;
}
public final String getTimezone()
{
public final String getTimezone() {
return timezone;
}
@@ -784,8 +739,7 @@ public class QueryLogEvent extends LogEvent
*
* @return Returns the charsetID.
*/
public final int getClientCharset()
{
public final int getClientCharset() {
return clientCharset;
}
@@ -794,8 +748,7 @@ public class QueryLogEvent extends LogEvent
*
* @return Returns the clientCollationId.
*/
public final int getClientCollation()
{
public final int getClientCollation() {
return clientCollation;
}
@@ -804,19 +757,16 @@ public class QueryLogEvent extends LogEvent
*
* @return Returns the serverCollationId.
*/
public final int getServerCollation()
{
public final int getServerCollation() {
return serverCollation;
}
/**
* Returns the sql_mode value.
*
* <p>
* The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
* and see mysql_priv.h for a list of the possible flags. Currently
* (2007-10-04), the following flags are available:
*
* <ul>
* <li>MODE_REAL_AS_FLOAT==0x1</li>
* <li>MODE_PIPES_AS_CONCAT==0x2</li>
@@ -851,14 +801,12 @@ public class QueryLogEvent extends LogEvent
* <li>MODE_NO_ENGINE_SUBSTITUTION=0x40000000</li>
* <li>MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000</li>
* </ul>
*
* All these flags are replicated from the server. However, all flags except
* MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always
* preserves its old value of MODE_NO_DIR_IN_CREATE. This field is always
* written to the binlog.
*/
public final long getSqlMode()
{
public final long getSqlMode() {
return sql_mode;
}
@@ -880,40 +828,35 @@ public class QueryLogEvent extends LogEvent
* Manual. This field is always written to the binlog in version >= 5.0, and
* never written in version < 5.0.
*/
public final long getFlags2()
{
public final long getFlags2() {
return flags2;
}
/**
* Returns the OPTION_AUTO_IS_NULL flag.
*/
public final boolean isAutoIsNull()
{
public final boolean isAutoIsNull() {
return ((flags2 & OPTION_AUTO_IS_NULL) == OPTION_AUTO_IS_NULL);
}
/**
* Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
*/
public final boolean isForeignKeyChecks()
{
public final boolean isForeignKeyChecks() {
return ((flags2 & OPTION_NO_FOREIGN_KEY_CHECKS) != OPTION_NO_FOREIGN_KEY_CHECKS);
}
/**
* Returns the OPTION_NOT_AUTOCOMMIT flag.
*/
public final boolean isAutocommit()
{
public final boolean isAutocommit() {
return ((flags2 & OPTION_NOT_AUTOCOMMIT) != OPTION_NOT_AUTOCOMMIT);
}
/**
* Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
*/
public final boolean isUniqueChecks()
{
public final boolean isUniqueChecks() {
return ((flags2 & OPTION_RELAXED_UNIQUE_CHECKS) != OPTION_RELAXED_UNIQUE_CHECKS);
}

32
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java
View File

@@ -7,30 +7,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1
* does not need it (it's repeatable again) so this event needn't be written in
* 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it
* does not cause bugs).
*
* The state of the random number generation consists of 128 bits, which are
* stored internally as two 64-bit numbers.
*
* Binary Format
*
* does not cause bugs). The state of the random number generation consists of
* 128 bits, which are stored internally as two 64-bit numbers. Binary Format
* The Post-Header for this event type is empty. The Body has two components:
*
* <table>
* <caption>Body for Rand_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>seed1</td>
* <td>8 byte unsigned integer</td>
* <td>64 bit random seed1.</td>
* </tr>
*
* <tr>
* <td>seed2</td>
* <td>8 byte unsigned integer</td>
@@ -41,19 +32,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class RandLogEvent extends LogEvent
{
public final class RandLogEvent extends LogEvent {
/**
* Fixed data part: Empty
*
* <p>
* Variable data part:
*
* <ul>
* <li>8 bytes. The value for the first seed.</li>
* <li>8 bytes. The value for the second seed.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final long seed1;
@@ -63,21 +51,17 @@ public final class RandLogEvent extends LogEvent
public static final int RAND_SEED1_OFFSET = 0;
public static final int RAND_SEED2_OFFSET = 8;
public RandLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public RandLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
/* The Post-Header is empty. The Variable Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[RAND_EVENT - 1]
+ RAND_SEED1_OFFSET);
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
+ RAND_SEED1_OFFSET);
seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET);
seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET);
}
public final String getQuery()
{
public final String getQuery() {
return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2;
}
}

47
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java
View File

@@ -4,57 +4,44 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* This will be deprecated when we move to using sequence ids.
*
* Binary Format
*
* The Post-Header has one component:
*
* This will be deprecated when we move to using sequence ids. Binary Format The
* Post-Header has one component:
* <table>
* <caption>Post-Header for Rotate_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>position</td>
* <td>8 byte integer</td>
* <td>The position within the binlog to rotate to.</td>
* </tr>
*
* </table>
*
* The Body has one component:
*
* <table>
* <caption>Body for Rotate_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>new_log</td>
* <td>variable length string without trailing zero, extending to the end of the
* event (determined by the length field of the Common-Header)</td>
* <td>Name of the binlog to rotate to.</td>
* </tr>
*
* </table>
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class RotateLogEvent extends LogEvent
{
public final class RotateLogEvent extends LogEvent {
/**
* Fixed data part:
*
* <ul>
* <li>8 bytes. The position of the first event in the next log file. Always
* contains the number 4 (meaning the next event starts at position 4 in the
@@ -62,14 +49,11 @@ public final class RotateLogEvent extends LogEvent
* is assumed to be 4.</li>
* </ul>
* <p>
*
* Variable data part:
*
* <ul>
* <li>The name of the next binary log. The filename is not null-terminated.
* Its length is the event size minus the size of the fixed parts.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final String filename;
@@ -91,21 +75,20 @@ public final class RotateLogEvent extends LogEvent
*
* @throws MySQLExtractException
*/
public RotateLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public RotateLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int headerSize = descriptionEvent.commonHeaderLen;
final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1];
buffer.position(headerSize + R_POS_OFFSET);
position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET)
position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf
// +
// R_POS_OFFSET)
final int filenameOffset = headerSize + postHeaderLen;
int filenameLen = buffer.limit() - filenameOffset;
if (filenameLen > FN_REFLEN - 1)
filenameLen = FN_REFLEN - 1;
if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1;
buffer.position(filenameOffset);
filename = buffer.getFixString(filenameLen);
}
@@ -114,8 +97,7 @@ public final class RotateLogEvent extends LogEvent
* Creates a new <code>Rotate_log_event</code> without log information. This
* is used to generate missing log rotation events.
*/
public RotateLogEvent(String filename)
{
public RotateLogEvent(String filename){
super(ROTATE_HEADER);
this.filename = filename;
@@ -125,21 +107,18 @@ public final class RotateLogEvent extends LogEvent
/**
* Creates a new <code>Rotate_log_event</code> without log information.
*/
public RotateLogEvent(String filename, final long position)
{
public RotateLogEvent(String filename, final long position){
super(ROTATE_HEADER);
this.filename = filename;
this.position = position;
}
public final String getFilename()
{
public final String getFilename() {
return filename;
}
public final long getPosition()
{
public final long getPosition() {
return position;
}
}

102
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
View File

@@ -12,41 +12,33 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public abstract class RowsLogEvent extends LogEvent
{
public abstract class RowsLogEvent extends LogEvent {
/**
* Fixed data part:
*
* <ul>
* <li>6 bytes. The table ID.</li>
* <li>2 bytes. Reserved for future use.</li>
* </ul>
*
* <p>
* Variable data part:
*
* <ul>
* <li>Packed integer. The number of columns in the table.</li>
*
* <li>Variable-sized. Bit-field indicating whether each column is used, one
* bit per column. For this field, the amount of storage required for N
* columns is INT((N+7)/8) bytes.</li>
*
* <li>Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating
* whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one
* bit per column. For this field, the amount of storage required for N
* columns is INT((N+7)/8) bytes.</li>
*
* <li>Variable-sized. A sequence of zero or more rows. The end is
* determined by the size of the event. Each row has the following format:
*
* <ul>
* <li>Variable-sized. Bit-field indicating whether each field in the row is
* NULL. Only columns that are "used" according to the second field in the
* variable data part are listed here. If the second field in the variable
* data part has N one-bits, the amount of storage required for this field
* is INT((N+7)/8) bytes.</li>
*
* <li>Variable-sized. The row-image, containing values of all table fields.
* This only lists table fields that are used (according to the second field
* of the variable data part) and non-NULL (according to the previous
@@ -54,16 +46,18 @@ public abstract class RowsLogEvent extends LogEvent
* number of zero bits in the previous field (not counting padding bits in
* the last byte). The format of each value is described in the
* log_event_print_value() function in log_event.cc.</li>
*
* <li>(for UPDATE_ROWS_EVENT only) the previous two fields are repeated,
* representing a second table row.</li>
* </ul>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final long tableId; /* Table ID */
private TableMapLogEvent table; /* The table the rows belong to */
private TableMapLogEvent table; /*
* The table
* the rows
* belong to
*/
/** Bitmap denoting columns available */
protected final int columnLen;
@@ -78,15 +72,17 @@ public abstract class RowsLogEvent extends LogEvent
protected final BitSet changeColumns;
/** XXX: Don't handle buffer in another thread. */
private final LogBuffer rowsBuf; /* The rows in packed format */
private final LogBuffer rowsBuf; /*
* The rows in
* packed
* format
*/
/**
* enum enum_flag
*
* These definitions allow you to combine the flags into an appropriate flag
* set using the normal bitwise operators. The implicit conversion from an
* enum-constant to an integer is accepted by the compiler, which is then
* used to set the real set of flags.
* enum enum_flag These definitions allow you to combine the flags into an
* appropriate flag set using the normal bitwise operators. The implicit
* conversion from an enum-constant to an integer is accepted by the
* compiler, which is then used to set the real set of flags.
*/
private final int flags;
@@ -112,43 +108,40 @@ public abstract class RowsLogEvent extends LogEvent
public static final int RW_V_TAG_LEN = 1;
public static final int RW_V_EXTRAINFO_TAG = 0;
public RowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
int headerLen = 0;
buffer.position(commonHeaderLen + RW_MAPID_OFFSET);
if (postHeaderLen == 6)
{
/* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
if (postHeaderLen == 6) {
/*
* Master is of an intermediate source tree before 5.1.4. Id is 4
* bytes
*/
tableId = buffer.getUint32();
}
else
{
} else {
tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET
}
flags = buffer.getUint16();
if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2)
{
if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2) {
headerLen = buffer.getUint16();
headerLen -= 2;
int start = buffer.position();
int end = start + headerLen;
for(int i = start ;i < end; ){
for (int i = start; i < end;) {
switch (buffer.getUint8(i++)) {
case RW_V_EXTRAINFO_TAG:
// int infoLen = buffer.getUint8();
buffer.position(i + EXTRA_ROW_INFO_LEN_OFFSET);
int checkLen = buffer.getUint8(); // EXTRA_ROW_INFO_LEN_OFFSET
int val= checkLen - EXTRA_ROW_INFO_HDR_BYTES;
assert(buffer.getUint8() == val); //EXTRA_ROW_INFO_FORMAT_OFFSET
for (int j= 0; j < val; j++) {
assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i
int val = checkLen - EXTRA_ROW_INFO_HDR_BYTES;
assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_FORMAT_OFFSET
for (int j = 0; j < val; j++) {
assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES
// + i
}
break;
default:
@@ -157,17 +150,14 @@ public abstract class RowsLogEvent extends LogEvent
}
}
}
buffer.position(commonHeaderLen + postHeaderLen + headerLen);
columnLen = (int) buffer.getPackedLong();
columns = buffer.getBitmap(columnLen);
if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT)
{
if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) {
changeColumns = buffer.getBitmap(columnLen);
}
else
{
} else {
changeColumns = columns;
}
@@ -176,46 +166,38 @@ public abstract class RowsLogEvent extends LogEvent
rowsBuf = buffer.duplicate(dataSize);
}
public final void fillTable(LogContext context)
{
public final void fillTable(LogContext context) {
table = context.getTable(tableId);
// end of statement check:
if ((flags & RowsLogEvent.STMT_END_F) != 0)
{
if ((flags & RowsLogEvent.STMT_END_F) != 0) {
// Now is safe to clear ignored map (clear_tables will also
// delete original table map events stored in the map).
context.clearAllTables();
}
}
public final long getTableId()
{
public final long getTableId() {
return tableId;
}
public final TableMapLogEvent getTable()
{
public final TableMapLogEvent getTable() {
return table;
}
public final BitSet getColumns()
{
public final BitSet getColumns() {
return columns;
}
public final BitSet getChangeColumns()
{
public final BitSet getChangeColumns() {
return changeColumns;
}
public final RowsLogBuffer getRowsBuf(String charsetName)
{
public final RowsLogBuffer getRowsBuf(String charsetName) {
return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName);
}
public final int getFlags(final int flags)
{
public final int getFlags(final int flags) {
return this.flags & flags;
}
}

22
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java
View File

@@ -5,21 +5,18 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and
* 4.x).
*
* Format_description_log_event derives from Start_log_event_v3; it is the
* 4.x). Format_description_log_event derives from Start_log_event_v3; it is the
* Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that
* describes the other events' Common-Header/Post-Header lengths. This event is
* sent by MySQL 5.0 whenever it starts sending a new binlog if the requested
* position is >4 (otherwise if ==4 the event will be sent naturally).
*
* @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class StartLogEventV3 extends LogEvent
{
public class StartLogEventV3 extends LogEvent {
/**
* We could have used SERVER_VERSION_LENGTH, but this introduces an obscure
* dependency - if somebody decided to change SERVER_VERSION_LENGTH this
@@ -34,9 +31,7 @@ public class StartLogEventV3 extends LogEvent
protected int binlogVersion;
protected String serverVersion;
public StartLogEventV3(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public StartLogEventV3(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
buffer.position(descriptionEvent.commonHeaderLen);
@@ -44,18 +39,15 @@ public class StartLogEventV3 extends LogEvent
serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
}
public StartLogEventV3()
{
public StartLogEventV3(){
super(new LogHeader(START_EVENT_V3));
}
public final String getServerVersion()
{
public final String getServerVersion() {
return serverVersion;
}
public final int getBinlogVersion()
{
public final int getBinlogVersion() {
return binlogVersion;
}
}

14
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java
View File

@@ -4,19 +4,15 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* Stop_log_event.
*
* The Post-Header and Body for this event type are empty; it only has the
* Common-Header.
* Stop_log_event. The Post-Header and Body for this event type are empty; it
* only has the Common-Header.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class StopLogEvent extends LogEvent
{
public StopLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent description_event)
{
public final class StopLogEvent extends LogEvent {
public StopLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent description_event){
super(header);
}
}

191
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
View File

@@ -8,44 +8,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* In row-based mode, every row operation event is preceded by a
* Table_map_log_event which maps a table definition to a number. The table
* definition consists of database name, table name, and column definitions.
*
* The Post-Header has the following components:
*
* definition consists of database name, table name, and column definitions. The
* Post-Header has the following components:
* <table>
* <caption>Post-Header for Table_map_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>table_id</td>
* <td>6 bytes unsigned integer</td>
* <td>The number that identifies the table.</td>
* </tr>
*
* <tr>
* <td>flags</td>
* <td>2 byte bitfield</td>
* <td>Reserved for future use; currently always 0.</td>
* </tr>
*
* </table>
*
* The Body has the following components:
*
* <table>
* <caption>Body for Table_map_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>database_name</td>
* <td>one byte string length, followed by null-terminated string</td>
@@ -55,20 +45,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* by a terminating 0 byte. (Note the redundancy in the representation of the
* length.)</td>
* </tr>
*
* <tr>
* <td>table_name</td>
* <td>one byte string length, followed by null-terminated string</td>
* <td>The name of the table, encoded the same way as the database name above.</td>
* </tr>
*
* <tr>
* <td>column_count</td>
* <td>packed_integer "Packed Integer"</td>
* <td>The number of columns in the table, represented as a packed
* variable-length integer.</td>
* </tr>
*
* <tr>
* <td>column_type</td>
* <td>List of column_count 1 byte enumeration values</td>
@@ -78,13 +65,11 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* listed in the table Table_table_map_log_event_column_types "below" (along
* with description of the associated metadata field).</td>
* </tr>
*
* <tr>
* <td>metadata_length</td>
* <td>packed_integer "Packed Integer"</td>
* <td>The length of the following metadata block</td>
* </tr>
*
* <tr>
* <td>metadata</td>
* <td>list of metadata for each column</td>
@@ -93,7 +78,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* metadata for each column are listed in the table
* Table_table_map_log_event_column_types "below".</td>
* </tr>
*
* <tr>
* <td>null_bits</td>
* <td>column_count bits, rounded up to nearest byte</td>
@@ -104,13 +88,10 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* first byte, the ninth is in the least significant bit of the second byte, and
* so on.</td>
* </tr>
*
* </table>
*
* The table below lists all column types, along with the numerical identifier
* for it and the size and interpretation of meta-data used to describe the
* type.
*
* <table>
* <caption>Table_map_log_event column types: numerical identifier and
* metadata</caption>
@@ -120,35 +101,30 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <th>Size of metadata in bytes</th>
* <th>Description of metadata</th>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DECIMAL</td>
* <td>0</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TINY</td>
* <td>1</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_SHORT</td>
* <td>2</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_LONG</td>
* <td>3</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_FLOAT</td>
* <td>4</td>
@@ -156,7 +132,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>1 byte unsigned integer, representing the "pack_length", which is equal
* to sizeof(float) on the server from which the event originates.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DOUBLE</td>
* <td>5</td>
@@ -164,63 +139,54 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>1 byte unsigned integer, representing the "pack_length", which is equal
* to sizeof(double) on the server from which the event originates.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_NULL</td>
* <td>6</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TIMESTAMP</td>
* <td>7</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_LONGLONG</td>
* <td>8</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_INT24</td>
* <td>9</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DATE</td>
* <td>10</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TIME</td>
* <td>11</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DATETIME</td>
* <td>12</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_YEAR</td>
* <td>13</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_NEWDATE</i></td>
* <td><i>14</i></td>
@@ -228,14 +194,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_VARCHAR</td>
* <td>15</td>
* <td>2 bytes</td>
* <td>2 byte unsigned integer representing the maximum length of the string.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_BIT</td>
* <td>16</td>
@@ -245,7 +209,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* occupied by the bitfield. The number of bytes is either int((length+7)/8) or
* int(length/8).</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_NEWDECIMAL</td>
* <td>246</td>
@@ -253,7 +216,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>A 1 byte unsigned int representing the precision, followed by a 1 byte
* unsigned int representing the number of decimals.</td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_ENUM</i></td>
* <td><i>247</i></td>
@@ -261,7 +223,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_SET</i></td>
* <td><i>248</i></td>
@@ -269,7 +230,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TINY_BLOB</td>
* <td>249</td>
@@ -277,7 +237,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_MEDIUM_BLOB</i></td>
* <td><i>250</i></td>
@@ -285,7 +244,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_LONG_BLOB</i></td>
* <td><i>251</i></td>
@@ -293,7 +251,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_BLOB</td>
* <td>252</td>
@@ -301,7 +258,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>The pack length, i.e., the number of bytes needed to represent the length
* of the blob: 1, 2, 3, or 4.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_VAR_STRING</td>
* <td>253</td>
@@ -312,7 +268,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* unsigned integer representing the field size, i.e., the number of bytes
* needed to store the length of the string.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_STRING</td>
* <td>254</td>
@@ -321,7 +276,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* byte is the field size, i.e., the number of bytes in the representation of
* size of the string: 3 or 4.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_GEOMETRY</td>
* <td>255</td>
@@ -329,21 +283,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>The pack length, i.e., the number of bytes needed to represent the length
* of the geometry: 1, 2, 3, or 4.</td>
* </tr>
*
* </table>
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class TableMapLogEvent extends LogEvent
{
public final class TableMapLogEvent extends LogEvent {
/**
* Fixed data part:
* <ul>
* <li>6 bytes. The table ID.</li>
* <li>2 bytes. Reserved for future use.</li>
* </ul>
*
* <p>
* Variable data part:
* <ul>
@@ -360,7 +312,6 @@ public final class TableMapLogEvent extends LogEvent
* one bit per column. For this field, the amount of storage required for N
* columns is INT((N+7)/8) bytes.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
protected final String dbname;
@@ -369,14 +320,15 @@ public final class TableMapLogEvent extends LogEvent
/**
* Holding mysql column information.
*/
public static final class ColumnInfo
{
public static final class ColumnInfo {
public int type;
public int meta;
}
protected final int columnCnt;
protected final ColumnInfo[] columnInfo; // buffer for field metadata
protected final ColumnInfo[] columnInfo; // buffer for field
// metadata
protected final long tableId;
protected BitSet nullBits;
@@ -388,22 +340,20 @@ public final class TableMapLogEvent extends LogEvent
/**
* Constructor used by slave to read the event from the binary log.
*/
public TableMapLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public TableMapLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
/* Read the post-header */
buffer.position(commonHeaderLen + TM_MAPID_OFFSET);
if (postHeaderLen == 6)
{
/* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
if (postHeaderLen == 6) {
/*
* Master is of an intermediate source tree before 5.1.4. Id is 4
* bytes
*/
tableId = buffer.getUint32();
}
else
{
} else {
// DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN);
tableId = buffer.getUlong48();
}
@@ -419,15 +369,13 @@ public final class TableMapLogEvent extends LogEvent
// Read column information from buffer
columnCnt = (int) buffer.getPackedLong();
columnInfo = new ColumnInfo[columnCnt];
for (int i = 0; i < columnCnt; i++)
{
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = new ColumnInfo();
info.type = buffer.getUint8();
columnInfo[i] = info;
}
if (buffer.position() < buffer.limit())
{
if (buffer.position() < buffer.limit()) {
final int fieldSize = (int) buffer.getPackedLong();
decodeFields(buffer, fieldSize);
nullBits = buffer.getBitmap(columnCnt);
@@ -439,41 +387,37 @@ public final class TableMapLogEvent extends LogEvent
*
* @see mysql-5.1.60/sql/rpl_utility.h
*/
private final void decodeFields(LogBuffer buffer, final int len)
{
private final void decodeFields(LogBuffer buffer, final int len) {
final int limit = buffer.limit();
buffer.limit(len + buffer.position());
for (int i = 0; i < columnCnt; i++)
{
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = columnInfo[i];
switch (info.type)
{
case MYSQL_TYPE_TINY_BLOB:
case MYSQL_TYPE_BLOB:
case MYSQL_TYPE_MEDIUM_BLOB:
case MYSQL_TYPE_LONG_BLOB:
case MYSQL_TYPE_DOUBLE:
case MYSQL_TYPE_FLOAT:
case MYSQL_TYPE_GEOMETRY:
/*
These types store a single byte.
*/
info.meta = buffer.getUint8();
break;
case MYSQL_TYPE_SET:
case MYSQL_TYPE_ENUM:
/*
* log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This
* enumeration value is only used internally and cannot
* exist in a binlog.
*/
logger.warn("This enumeration value is only used internally "
+ "and cannot exist in a binlog: type=" + info.type);
break;
case MYSQL_TYPE_STRING:
{
switch (info.type) {
case MYSQL_TYPE_TINY_BLOB:
case MYSQL_TYPE_BLOB:
case MYSQL_TYPE_MEDIUM_BLOB:
case MYSQL_TYPE_LONG_BLOB:
case MYSQL_TYPE_DOUBLE:
case MYSQL_TYPE_FLOAT:
case MYSQL_TYPE_GEOMETRY:
/*
* These types store a single byte.
*/
info.meta = buffer.getUint8();
break;
case MYSQL_TYPE_SET:
case MYSQL_TYPE_ENUM:
/*
* log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This
* enumeration value is only used internally and cannot
* exist in a binlog.
*/
logger.warn("This enumeration value is only used internally "
+ "and cannot exist in a binlog: type=" + info.type);
break;
case MYSQL_TYPE_STRING: {
/*
* log_event.h : The first byte is always
* MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the
@@ -485,59 +429,52 @@ public final class TableMapLogEvent extends LogEvent
info.meta = x;
break;
}
case MYSQL_TYPE_BIT:
info.meta = buffer.getUint16();
break;
case MYSQL_TYPE_VARCHAR:
/*
* These types store two bytes.
*/
info.meta = buffer.getUint16();
break;
case MYSQL_TYPE_NEWDECIMAL:
{
case MYSQL_TYPE_BIT:
info.meta = buffer.getUint16();
break;
case MYSQL_TYPE_VARCHAR:
/*
* These types store two bytes.
*/
info.meta = buffer.getUint16();
break;
case MYSQL_TYPE_NEWDECIMAL: {
int x = buffer.getUint8() << 8; // precision
x += buffer.getUint8(); // decimals
info.meta = x;
break;
}
case MYSQL_TYPE_TIME2:
case MYSQL_TYPE_DATETIME2:
case MYSQL_TYPE_TIMESTAMP2:
{
case MYSQL_TYPE_TIME2:
case MYSQL_TYPE_DATETIME2:
case MYSQL_TYPE_TIMESTAMP2: {
info.meta = buffer.getUint8();
break;
}
default:
info.meta = 0;
break;
default:
info.meta = 0;
break;
}
}
buffer.limit(limit);
}
public final String getDbName()
{
public final String getDbName() {
return dbname;
}
public final String getTableName()
{
public final String getTableName() {
return tblname;
}
public final int getColumnCnt()
{
public final int getColumnCnt() {
return columnCnt;
}
public final ColumnInfo[] getColumnInfo()
{
public final ColumnInfo[] getColumnInfo() {
return columnInfo;
}
public final long getTableId()
{
public final long getTableId() {
return tableId;
}
}

7
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java
View File

@@ -8,10 +8,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class UnknownLogEvent extends LogEvent
{
public UnknownLogEvent(LogHeader header)
{
public final class UnknownLogEvent extends LogEvent {
public UnknownLogEvent(LogHeader header){
super(header);
}
}

15
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java
View File

@@ -4,19 +4,16 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
/**
* Log row updates with a before image. The event contain several update rows
* for a table. Note that each event contains only rows for one table.
*
* Also note that the row data consists of pairs of row data: one row for the
* old data and one row for the new data.
* for a table. Note that each event contains only rows for one table. Also note
* that the row data consists of pairs of row data: one row for the old data and
* one row for the new data.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class UpdateRowsLogEvent extends RowsLogEvent
{
public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class UpdateRowsLogEvent extends RowsLogEvent {
public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

99
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java
View File

@@ -8,22 +8,19 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* User_var_log_event.
*
* Every time a query uses the value of a user variable, a User_var_log_event is
* written before the Query_log_event, to set the user variable.
* User_var_log_event. Every time a query uses the value of a user variable, a
* User_var_log_event is written before the Query_log_event, to set the user
* variable.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class UserVarLogEvent extends LogEvent
{
public final class UserVarLogEvent extends LogEvent {
/**
* Fixed data part: Empty
*
* <p>
* Variable data part:
*
* <ul>
* <li>4 bytes. the size of the user variable name.</li>
* <li>The user variable name.</li>
@@ -39,7 +36,6 @@ public final class UserVarLogEvent extends LogEvent
* <li>Variable-sized. For a string variable, this is the string. For a
* float or integer variable, this is its value in 8 bytes.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final String name;
@@ -66,80 +62,65 @@ public final class UserVarLogEvent extends LogEvent
public static final int UV_NAME_LEN_SIZE = 4;
public static final int UV_CHARSET_NUMBER_SIZE = 4;
public UserVarLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent) throws IOException
{
public UserVarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
super(header);
/* The Post-Header is empty. The Variable Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
final int nameLen = (int) buffer.getUint32();
name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE
isNull = (0 != buffer.getInt8());
if (isNull)
{
if (isNull) {
type = STRING_RESULT;
charsetNumber = 63; /* binary */
value = null;
}
else
{
} else {
type = buffer.getInt8(); // UV_VAL_IS_NULL
charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE
final int valueLen = (int) buffer.getUint32(); // buf + UV_CHARSET_NUMBER_SIZE
final int valueLen = (int) buffer.getUint32(); // buf +
// UV_CHARSET_NUMBER_SIZE
final int limit = buffer.limit(); /* for restore */
buffer.limit(buffer.position() + valueLen);
/* @see User_var_log_event::print */
switch (type)
{
case REAL_RESULT:
value = Double.valueOf(buffer.getDouble64()); // float8get
break;
case INT_RESULT:
if (valueLen == 8)
value = Long.valueOf(buffer.getLong64()); // !uint8korr
else if (valueLen == 4)
value = Long.valueOf(buffer.getUint32());
else
throw new IOException("Error INT_RESULT length: "
+ valueLen);
break;
case DECIMAL_RESULT:
final int precision = buffer.getInt8();
final int scale = buffer.getInt8();
value = buffer.getDecimal(precision, scale); // bin2decimal
break;
case STRING_RESULT:
String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
value = buffer.getFixString(valueLen, charsetName);
break;
case ROW_RESULT:
// this seems to be banned in MySQL altogether
throw new IOException("ROW_RESULT is unsupported");
default:
value = null;
break;
switch (type) {
case REAL_RESULT:
value = Double.valueOf(buffer.getDouble64()); // float8get
break;
case INT_RESULT:
if (valueLen == 8) value = Long.valueOf(buffer.getLong64()); // !uint8korr
else if (valueLen == 4) value = Long.valueOf(buffer.getUint32());
else throw new IOException("Error INT_RESULT length: " + valueLen);
break;
case DECIMAL_RESULT:
final int precision = buffer.getInt8();
final int scale = buffer.getInt8();
value = buffer.getDecimal(precision, scale); // bin2decimal
break;
case STRING_RESULT:
String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
value = buffer.getFixString(valueLen, charsetName);
break;
case ROW_RESULT:
// this seems to be banned in MySQL altogether
throw new IOException("ROW_RESULT is unsupported");
default:
value = null;
break;
}
buffer.limit(limit);
}
}
public final String getQuery()
{
if (value == null)
{
public final String getQuery() {
if (value == null) {
return "SET @" + name + " := NULL";
}
else if (type == STRING_RESULT)
{
} else if (type == STRING_RESULT) {
// TODO: do escaping !?
return "SET @" + name + " := \'" + value + '\'';
}
else
{
} else {
return "SET @" + name + " := " + String.valueOf(value);
}
}

8
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java
View File

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class WriteRowsLogEvent extends RowsLogEvent
{
public WriteRowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class WriteRowsLogEvent extends RowsLogEvent {
public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

14
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java
View File

@@ -10,23 +10,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class XidLogEvent extends LogEvent
{
public final class XidLogEvent extends LogEvent {
private final long xid;
public XidLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public XidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
/* The Post-Header is empty. The Variable Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[XID_EVENT - 1]);
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
xid = buffer.getLong64(); // !uint8korr
}
public final long getXid()
{
public final long getXid() {
return xid;
}
}

34
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
View File

@@ -10,8 +10,8 @@ import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
public class BaseLogFetcherTest {
@@ -20,36 +20,38 @@ public class BaseLogFetcherTest {
protected Charset charset = Charset.forName("utf-8");
protected void parseQueryEvent(QueryLogEvent event) {
System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getCatalog()));
System.out.println(String.format("================> binlog[%s:%s] , name[%s]",
binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getCatalog()));
System.out.println("sql : " + event.getQuery());
}
protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
.getLogPos() - event.getHeader().getEventLen()));
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
}
protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
.getLogPos() - event.getHeader().getEventLen()));
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
}
protected void parseXidEvent(XidLogEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
.getLogPos() - event.getHeader().getEventLen()));
System.out.println("xid : " + event.getXid());
}
protected void parseRowsEvent(RowsLogEvent event) {
try {
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getTable().getDbName(), event.getTable().getTableName()));
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getTable().getDbName(),
event.getTable().getTableName()));
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();

11
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java
View File

@@ -48,23 +48,22 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
binlogFileName = ((RotateLogEvent)
event).getFilename();
binlogFileName = ((RotateLogEvent) event).getFilename();
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
parseRowsEvent((WriteRowsLogEvent) event);
parseRowsEvent((WriteRowsLogEvent) event);
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT:
parseRowsEvent((UpdateRowsLogEvent) event);
parseRowsEvent((UpdateRowsLogEvent) event);
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
parseRowsEvent((DeleteRowsLogEvent) event);
parseRowsEvent((DeleteRowsLogEvent) event);
break;
case LogEvent.QUERY_EVENT:
parseQueryEvent((QueryLogEvent) event);
parseQueryEvent((QueryLogEvent) event);
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
parseRowsQueryEvent((RowsQueryLogEvent) event);

144
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java
View File

@@ -5,14 +5,11 @@ import java.math.BigInteger;
import junit.framework.TestCase;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
public class LogBufferTest extends TestCase {
public class LogBufferTest extends TestCase
{
public static final int LOOP = 10000;
public void testSigned()
{
public void testSigned() {
byte[] array = { 0, 0, 0, (byte) 0xff };
LogBuffer buffer = new LogBuffer(array, 0, array.length);
@@ -24,17 +21,15 @@ public class LogBufferTest extends TestCase
System.out.println(buffer.getUint24(1));
}
public void testBigInteger()
{
byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
public void testBigInteger() {
byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff };
LogBuffer buffer = new LogBuffer(array, 0, array.length);
long tt1 = 0;
long l1 = 0;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t1 = System.nanoTime();
l1 = buffer.getLong64(0);
tt1 += System.nanoTime() - t1;
@@ -45,8 +40,7 @@ public class LogBufferTest extends TestCase
long tt2 = 0;
BigInteger l2 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t2 = System.nanoTime();
l2 = buffer.getUlong64(0);
tt2 += System.nanoTime() - t2;
@@ -57,18 +51,15 @@ public class LogBufferTest extends TestCase
}
/* Reads big-endian integer from no more than 4 bytes */
private static int convertNBytesToInt(byte[] buffer, int offset, int length)
{
private static int convertNBytesToInt(byte[] buffer, int offset, int length) {
int ret = 0;
for (int i = offset; i < (offset + length); i++)
{
for (int i = offset; i < (offset + length); i++) {
ret = (ret << 8) | (0xff & buffer[i]);
}
return ret;
}
private static int convert4BytesToInt(byte[] buffer, int offset)
{
private static int convert4BytesToInt(byte[] buffer, int offset) {
int value;
value = (0xff & buffer[offset + 3]);
value += (0xff & buffer[offset + 2]) << 8;
@@ -77,24 +68,20 @@ public class LogBufferTest extends TestCase
return value;
}
public static short convert1ByteToShort(byte[] buffer, int offset)
{
public static short convert1ByteToShort(byte[] buffer, int offset) {
short value;
value = (short) buffer[offset + 0];
return value;
}
public static short convert2bytesToShort(byte[] buffer, int offset)
{
public static short convert2bytesToShort(byte[] buffer, int offset) {
short value;
value = (short) (buffer[offset + 0] << 8);
value += (short) (buffer[offset + 1] & 0xff);
return value;
}
public static final BigDecimal extractDecimal(byte[] buffer, int precision,
int scale)
{
public static final BigDecimal extractDecimal(byte[] buffer, int precision, int scale) {
//
// Decimal representation in binlog seems to be as follows:
// 1 byte - 'precision'
@@ -113,7 +100,7 @@ public class LogBufferTest extends TestCase
// 0x00000005 0x1b38b060 0x00
// 5 456700000 0
// 54567000000 / 10^{10} = 5.4567
//
//
// int_size below shows how long is integer part
//
// offset = offset + 2; // offset of the number part
@@ -129,34 +116,28 @@ public class LogBufferTest extends TestCase
int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
// how many bytes are used to represent given amount of digits?
int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
+ LogBuffer.dig2bytes[intg0x];
int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
+ LogBuffer.dig2bytes[frac0x];
int integerSize = intg0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[intg0x];
int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[frac0x];
int bin_size = integerSize + decimalSize; // total bytes
byte[] d_copy = new byte[bin_size];
if (bin_size > buffer.length)
{
throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
+ bin_size + ", available bytes: " + buffer.length);
if (bin_size > buffer.length) {
throw new ArrayIndexOutOfBoundsException("Calculated bin_size: " + bin_size + ", available bytes: "
+ buffer.length);
}
// Invert first bit
d_copy[0] = buffer[0];
d_copy[0] ^= 0x80;
if (sign == -1)
{
if (sign == -1) {
// Invert every byte
d_copy[0] ^= 0xFF;
}
for (int i = 1; i < bin_size; i++)
{
for (int i = 1; i < bin_size; i++) {
d_copy[i] = buffer[i];
if (sign == -1)
{
if (sign == -1) {
// Invert every byte
d_copy[i] ^= 0xFF;
}
@@ -167,62 +148,49 @@ public class LogBufferTest extends TestCase
BigDecimal intPart = new BigDecimal(0);
if (offset > 0)
intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
if (offset > 0) intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
while (offset < integerSize)
{
intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
while (offset < integerSize) {
intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(BigDecimal.valueOf(convert4BytesToInt(d_copy,
offset)));
offset += 4;
}
// Decimal part
BigDecimal fracPart = new BigDecimal(0);
int shift = 0;
for (int i = 0; i < frac0; i++)
{
for (int i = 0; i < frac0; i++) {
shift += LogBuffer.DIG_PER_DEC1;
fracPart = fracPart.add(BigDecimal.valueOf(
convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
fracPart = fracPart.add(BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
offset += 4;
}
if (LogBuffer.dig2bytes[frac0x] > 0)
{
fracPart = fracPart.add(BigDecimal.valueOf(
convertNBytesToInt(d_copy, offset,
LogBuffer.dig2bytes[frac0x])).movePointLeft(
shift + frac0x));
if (LogBuffer.dig2bytes[frac0x] > 0) {
fracPart = fracPart.add(BigDecimal.valueOf(convertNBytesToInt(d_copy, offset, LogBuffer.dig2bytes[frac0x]))
.movePointLeft(shift + frac0x));
}
return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
}
public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
0x38, (byte) 0xb0, 0x60, 0x00 };
public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b, 0x38, (byte) 0xb0, 0x60, 0x00 };
public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
(byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
(byte) 0xa0, (byte) 0xff };
public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7,
(byte) 0x4f, (byte) 0xa0, (byte) 0xff };
public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
-108 };
public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96, -108 };
public void testBigDecimal() throws InterruptedException
{
do
{
public void testBigDecimal() throws InterruptedException {
do {
System.out.println("old extract decimal: ");
long tt1 = 0;
BigDecimal bd1 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t1 = System.nanoTime();
bd1 = extractDecimal(array2, 19, 10);
tt1 += System.nanoTime() - t1;
@@ -233,8 +201,7 @@ public class LogBufferTest extends TestCase
long tt2 = 0;
BigDecimal bd2 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t2 = System.nanoTime();
bd2 = extractDecimal(array1, 19, 10);
tt2 += System.nanoTime() - t2;
@@ -245,8 +212,7 @@ public class LogBufferTest extends TestCase
long tt3 = 0;
BigDecimal bd3 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t3 = System.nanoTime();
bd3 = extractDecimal(array3, 18, 6);
tt3 += System.nanoTime() - t3;
@@ -257,8 +223,7 @@ public class LogBufferTest extends TestCase
long tt4 = 0;
BigDecimal bd4 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t4 = System.nanoTime();
bd4 = extractDecimal(array4, 18, 6);
tt4 += System.nanoTime() - t4;
@@ -269,8 +234,7 @@ public class LogBufferTest extends TestCase
long tt5 = 0;
BigDecimal bd5 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t5 = System.nanoTime();
bd5 = extractDecimal(array5, 18, 6);
tt5 += System.nanoTime() - t5;
@@ -278,11 +242,9 @@ public class LogBufferTest extends TestCase
System.out.print(tt5 / LOOP);
System.out.print("ns >> ");
System.out.println(bd5);
}
while (false);
} while (false);
do
{
do {
System.out.println("new extract decimal: ");
LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
@@ -293,8 +255,7 @@ public class LogBufferTest extends TestCase
long tt1 = 0;
BigDecimal bd1 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t1 = System.nanoTime();
bd1 = buffer1.getDecimal(0, 19, 10);
tt1 += System.nanoTime() - t1;
@@ -305,8 +266,7 @@ public class LogBufferTest extends TestCase
long tt2 = 0;
BigDecimal bd2 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t2 = System.nanoTime();
bd2 = buffer2.getDecimal(0, 19, 10);
tt2 += System.nanoTime() - t2;
@@ -317,8 +277,7 @@ public class LogBufferTest extends TestCase
long tt3 = 0;
BigDecimal bd3 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t3 = System.nanoTime();
bd3 = buffer3.getDecimal(0, 18, 6);
tt3 += System.nanoTime() - t3;
@@ -329,8 +288,7 @@ public class LogBufferTest extends TestCase
long tt4 = 0;
BigDecimal bd4 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t4 = System.nanoTime();
bd4 = buffer4.getDecimal(0, 18, 6);
tt4 += System.nanoTime() - t4;
@@ -341,8 +299,7 @@ public class LogBufferTest extends TestCase
long tt5 = 0;
BigDecimal bd5 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t5 = System.nanoTime();
bd5 = buffer5.getDecimal(0, 18, 6);
tt5 += System.nanoTime() - t5;
@@ -350,7 +307,6 @@ public class LogBufferTest extends TestCase
System.out.print(tt5 / LOOP);
System.out.print("ns >> ");
System.out.println(bd5);
}
while (false);
} while (false);
}
}