chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

294
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java Normal file
View File

@@ -0,0 +1,294 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.ClientAuthenticationPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.HandshakeInitializationPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.Reply323Packet;
import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
/**
* 基于mysql socket协议的链接实现
*
* @author jianghang 2013-2-18 下午09:22:30
* @version 1.0.1
*/
public class MysqlConnector {
private static final Logger logger = LoggerFactory.getLogger(MysqlConnector.class);
private InetSocketAddress address;
private String username;
private String password;
private byte charsetNumber = 33;
private String defaultSchema = "retl";
private int soTimeout = 30 * 1000;
private int receiveBufferSize = 16 * 1024;
private int sendBufferSize = 16 * 1024;
private SocketChannel channel;
private AtomicBoolean connected = new AtomicBoolean(false);
public MysqlConnector(){
}
public MysqlConnector(InetSocketAddress address, String username, String password){
this.address = address;
this.username = username;
this.password = password;
}
public MysqlConnector(InetSocketAddress address, String username, String password, byte charsetNumber,
String defaultSchema){
this(address, username, password);
this.charsetNumber = charsetNumber;
this.defaultSchema = defaultSchema;
}
public void connect() throws IOException {
if (connected.compareAndSet(false, true)) {
try {
channel = SocketChannel.open();
configChannel(channel);
logger.info("connect MysqlConnection to {}...", address);
channel.connect(address);
negotiate(channel);
} catch (Exception e) {
disconnect();
throw new IOException("connect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e));
}
} else {
logger.error("the channel can't be connected twice.");
}
}
public void reconnect() throws IOException {
disconnect();
connect();
}
public void disconnect() throws IOException {
if (connected.compareAndSet(true, false)) {
try {
if (channel != null) {
channel.close();
}
logger.info("disConnect MysqlConnection to {}...", address);
} catch (Exception e) {
throw new IOException("disconnect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e));
}
} else {
logger.info("the channel {} is not connected", this.address);
}
}
public boolean isConnected() {
return this.channel != null && this.channel.isConnected();
}
public MysqlConnector fork() {
MysqlConnector connector = new MysqlConnector();
connector.setCharsetNumber(getCharsetNumber());
connector.setDefaultSchema(getDefaultSchema());
connector.setAddress(getAddress());
connector.setPassword(password);
connector.setUsername(getUsername());
connector.setReceiveBufferSize(getReceiveBufferSize());
connector.setSendBufferSize(getSendBufferSize());
connector.setSoTimeout(getSoTimeout());
return connector;
}
// ====================== help method ====================
private void configChannel(SocketChannel channel) throws IOException {
channel.socket().setKeepAlive(true);
channel.socket().setReuseAddress(true);
channel.socket().setSoTimeout(soTimeout);
channel.socket().setTcpNoDelay(true);
channel.socket().setReceiveBufferSize(receiveBufferSize);
channel.socket().setSendBufferSize(sendBufferSize);
}
private void negotiate(SocketChannel channel) throws IOException {
HeaderPacket header = PacketManager.readHeader(channel, 4);
byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength());
if (body[0] < 0) {// check field_count
if (body[0] == -1) {
ErrorPacket error = new ErrorPacket();
error.fromBytes(body);
throw new IOException("handshake exception:\n" + error.toString());
} else if (body[0] == -2) {
throw new IOException("Unexpected EOF packet at handshake phase.");
} else {
throw new IOException("unpexpected packet with field_count=" + body[0]);
}
}
HandshakeInitializationPacket handshakePacket = new HandshakeInitializationPacket();
handshakePacket.fromBytes(body);
logger.info("handshake initialization packet received, prepare the client authentication packet to send");
ClientAuthenticationPacket clientAuth = new ClientAuthenticationPacket();
clientAuth.setCharsetNumber(charsetNumber);
clientAuth.setUsername(username);
clientAuth.setPassword(password);
clientAuth.setServerCapabilities(handshakePacket.serverCapabilities);
clientAuth.setDatabaseName(defaultSchema);
clientAuth.setScrumbleBuff(joinAndCreateScrumbleBuff(handshakePacket));
byte[] clientAuthPkgBody = clientAuth.toBytes();
HeaderPacket h = new HeaderPacket();
h.setPacketBodyLength(clientAuthPkgBody.length);
h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
PacketManager.write(channel,
new ByteBuffer[] { ByteBuffer.wrap(h.toBytes()), ByteBuffer.wrap(clientAuthPkgBody) });
logger.info("client authentication packet is sent out.");
// check auth result
header = null;
header = PacketManager.readHeader(channel, 4);
body = null;
body = PacketManager.readBytes(channel, header.getPacketBodyLength());
assert body != null;
if (body[0] < 0) {
if (body[0] == -1) {
ErrorPacket err = new ErrorPacket();
err.fromBytes(body);
throw new IOException("Error When doing Client Authentication:" + err.toString());
} else if (body[0] == -2) {
auth323(channel, header.getPacketSequenceNumber(), handshakePacket.seed);
// throw new
// IOException("Unexpected EOF packet at Client Authentication.");
} else {
throw new IOException("unpexpected packet with field_count=" + body[0]);
}
}
}
private void auth323(SocketChannel channel, byte packetSequenceNumber, byte[] seed) throws IOException {
// auth 323
Reply323Packet r323 = new Reply323Packet();
if (password != null && password.length() > 0) {
r323.seed = MySQLPasswordEncrypter.scramble323(password, new String(seed)).getBytes();
}
byte[] b323Body = r323.toBytes();
HeaderPacket h323 = new HeaderPacket();
h323.setPacketBodyLength(b323Body.length);
h323.setPacketSequenceNumber((byte) (packetSequenceNumber + 1));
PacketManager.write(channel, new ByteBuffer[] { ByteBuffer.wrap(h323.toBytes()), ByteBuffer.wrap(b323Body) });
logger.info("client 323 authentication packet is sent out.");
// check auth result
HeaderPacket header = PacketManager.readHeader(channel, 4);
byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength());
assert body != null;
switch (body[0]) {
case 0:
break;
case -1:
ErrorPacket err = new ErrorPacket();
err.fromBytes(body);
throw new IOException("Error When doing Client Authentication:" + err.toString());
default:
throw new IOException("unpexpected packet with field_count=" + body[0]);
}
}
private byte[] joinAndCreateScrumbleBuff(HandshakeInitializationPacket handshakePacket) throws IOException {
byte[] dest = new byte[handshakePacket.seed.length + handshakePacket.restOfScrambleBuff.length];
System.arraycopy(handshakePacket.seed, 0, dest, 0, handshakePacket.seed.length);
System.arraycopy(handshakePacket.restOfScrambleBuff,
0,
dest,
handshakePacket.seed.length,
handshakePacket.restOfScrambleBuff.length);
return dest;
}
public InetSocketAddress getAddress() {
return address;
}
public void setAddress(InetSocketAddress address) {
this.address = address;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public byte getCharsetNumber() {
return charsetNumber;
}
public void setCharsetNumber(byte charsetNumber) {
this.charsetNumber = charsetNumber;
}
public String getDefaultSchema() {
return defaultSchema;
}
public void setDefaultSchema(String defaultSchema) {
this.defaultSchema = defaultSchema;
}
public int getSoTimeout() {
return soTimeout;
}
public void setSoTimeout(int soTimeout) {
this.soTimeout = soTimeout;
}
public int getReceiveBufferSize() {
return receiveBufferSize;
}
public void setReceiveBufferSize(int receiveBufferSize) {
this.receiveBufferSize = receiveBufferSize;
}
public int getSendBufferSize() {
return sendBufferSize;
}
public void setSendBufferSize(int sendBufferSize) {
this.sendBufferSize = sendBufferSize;
}
public SocketChannel getChannel() {
return channel;
}
public void setChannel(SocketChannel channel) {
this.channel = channel;
}
public void setPassword(String password) {
this.password = password;
}
}

107
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java Normal file
View File

@@ -0,0 +1,107 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import java.io.IOException;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.RowDataPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
/**
* 默认输出的数据编码为UTF-8，如有需要请正确转码
*
* @author jianghang 2013-9-4 上午11:50:26
* @since 1.0.0
*/
public class MysqlQueryExecutor {
private SocketChannel channel;
public MysqlQueryExecutor(MysqlConnector connector){
if (!connector.isConnected()) {
throw new RuntimeException("should execute connector.connect() first");
}
this.channel = connector.getChannel();
}
public MysqlQueryExecutor(SocketChannel ch){
this.channel = ch;
}
/**
* (Result Set Header Packet) the number of columns <br>
* (Field Packets) column descriptors <br>
* (EOF Packet) marker: end of Field Packets <br>
* (Row Data Packets) row contents <br>
* (EOF Packet) marker: end of Data Packets
*
* @param queryString
* @return
* @throws IOException
*/
public ResultSetPacket query(String queryString) throws IOException {
QueryCommandPacket cmd = new QueryCommandPacket();
cmd.setQueryString(queryString);
byte[] bodyBytes = cmd.toBytes();
PacketManager.write(channel, bodyBytes);
byte[] body = readNextPacket();
if (body[0] < 0) {
ErrorPacket packet = new ErrorPacket();
packet.fromBytes(body);
throw new IOException(packet + "\n with command: " + queryString);
}
ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
rsHeader.fromBytes(body);
List<FieldPacket> fields = new ArrayList<FieldPacket>();
for (int i = 0; i < rsHeader.getColumnCount(); i++) {
FieldPacket fp = new FieldPacket();
fp.fromBytes(readNextPacket());
fields.add(fp);
}
readEofPacket();
List<RowDataPacket> rowData = new ArrayList<RowDataPacket>();
while (true) {
body = readNextPacket();
if (body[0] == -2) {
break;
}
RowDataPacket rowDataPacket = new RowDataPacket();
rowDataPacket.fromBytes(body);
rowData.add(rowDataPacket);
}
ResultSetPacket resultSet = new ResultSetPacket();
resultSet.getFieldDescriptors().addAll(fields);
for (RowDataPacket r : rowData) {
resultSet.getFieldValues().addAll(r.getColumns());
}
resultSet.setSourceAddress(channel.socket().getRemoteSocketAddress());
return resultSet;
}
private void readEofPacket() throws IOException {
byte[] eofBody = readNextPacket();
if (eofBody[0] != -2) {
throw new IOException("EOF Packet is expected, but packet with field_count=" + eofBody[0] + " is found.");
}
}
protected byte[] readNextPacket() throws IOException {
HeaderPacket h = PacketManager.readHeader(channel, 4);
return PacketManager.readBytes(channel, h.getPacketBodyLength());
}
}

56
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java Normal file
View File

@@ -0,0 +1,56 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import java.io.IOException;
import java.nio.channels.SocketChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.OKPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
/**
* 默认输出的数据编码为UTF-8，如有需要请正确转码
*
* @author jianghang 2013-9-4 上午11:51:11
* @since 1.0.0
*/
public class MysqlUpdateExecutor {
private static final Logger logger = LoggerFactory.getLogger(MysqlUpdateExecutor.class);
private SocketChannel channel;
public MysqlUpdateExecutor(MysqlConnector connector){
if (!connector.isConnected()) {
throw new RuntimeException("should execute connector.connect() first");
}
this.channel = connector.getChannel();
}
public MysqlUpdateExecutor(SocketChannel ch){
this.channel = ch;
}
public OKPacket update(String updateString) throws IOException {
QueryCommandPacket cmd = new QueryCommandPacket();
cmd.setQueryString(updateString);
byte[] bodyBytes = cmd.toBytes();
PacketManager.write(channel, bodyBytes);
logger.debug("read update result...");
byte[] body = PacketManager.readBytes(channel, PacketManager.readHeader(channel, 4).getPacketBodyLength());
if (body[0] < 0) {
ErrorPacket packet = new ErrorPacket();
packet.fromBytes(body);
throw new IOException(packet + "\n with command: " + updateString);
}
OKPacket packet = new OKPacket();
packet.fromBytes(body);
return packet;
}
}

24
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/CommandPacket.java Normal file
View File

@@ -0,0 +1,24 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
public abstract class CommandPacket implements IPacket {
private byte command;
// arg
public void setCommand(byte command) {
this.command = command;
}
public byte getCommand() {
return command;
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

73
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java Normal file
View File

@@ -0,0 +1,73 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* <pre>
* Offset Length Description
* 0 3 Packet body length stored with the low byte first.
* 3 1 Packet sequence number. The sequence numbers are reset with each new command.
* While the correct packet sequencing is ensured by the underlying transmission protocol,
* this field is used for the sanity checks of the application logic.
* </pre>
*
* <br>
* The Packet Header will not be shown in the descriptions of packets that follow this section. Think of it as always
* there. But logically, it "precedes the packet" rather than "is included in the packet".<br>
*
* @author fujohnwang
*/
public class HeaderPacket implements IPacket {
/**
* this field indicates the packet length that follows the header, with header packet's 4 bytes excluded.
*/
private int packetBodyLength;
private byte packetSequenceNumber;
/**
* little-endian byte order
*/
public byte[] toBytes() {
byte[] data = new byte[4];
data[0] = (byte) (packetBodyLength & 0xFF);
data[1] = (byte) (packetBodyLength >>> 8);
data[2] = (byte) (packetBodyLength >>> 16);
data[3] = getPacketSequenceNumber();
return data;
}
/**
* little-endian byte order
*/
public void fromBytes(byte[] data) {
if (data == null || data.length != 4) {
throw new IllegalArgumentException("invalid header data. It can't be null and the length must be 4 byte.");
}
this.packetBodyLength = (data[0] & 0xFF) | ((data[1] & 0xFF) << 8) | ((data[2] & 0xFF) << 16);
this.setPacketSequenceNumber(data[3]);
}
public int getPacketBodyLength() {
return packetBodyLength;
}
public void setPacketBodyLength(int packetBodyLength) {
this.packetBodyLength = packetBodyLength;
}
public void setPacketSequenceNumber(byte packetSequenceNumber) {
this.packetSequenceNumber = packetSequenceNumber;
}
public byte getPacketSequenceNumber() {
return packetSequenceNumber;
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

29
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java Normal file
View File

@@ -0,0 +1,29 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import java.io.IOException;
/**
* Top Abstraction for network packet.<br>
* it exposes 2 behaviors for sub-class implementation which will be used to
* marshal data into bytes before sending and to un-marshal data from data after
* receiving.<br>
*
* @author fujohnwang
* @see 1.0
*/
public interface IPacket {
/**
* un-marshal raw bytes into {@link IPacket} state for application usage.<br>
*
* @param data, the raw byte data received from networking
*/
void fromBytes(byte[] data) throws IOException;
/**
* marshal the {@link IPacket} state into raw bytes for sending out to
* network.<br>
*
* @return the bytes that's collected from {@link IPacket} state
*/
byte[] toBytes() throws IOException;
}

32
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/PacketWithHeaderPacket.java Normal file
View File

@@ -0,0 +1,32 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
import com.google.common.base.Preconditions;
public abstract class PacketWithHeaderPacket implements IPacket {
protected HeaderPacket header;
protected PacketWithHeaderPacket(){
}
protected PacketWithHeaderPacket(HeaderPacket header){
setHeader(header);
}
public void setHeader(HeaderPacket header) {
Preconditions.checkNotNull(header);
this.header = header;
}
public HeaderPacket getHeader() {
return header;
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

70
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java Normal file
View File

@@ -0,0 +1,70 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
/**
* COM_BINLOG_DUMP
*
* @author fujohnwang
* @since 1.0
*/
public class BinlogDumpCommandPacket extends CommandPacket {
/** BINLOG_DUMP options */
public static final int BINLOG_DUMP_NON_BLOCK = 1;
public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
public long binlogPosition;
public long slaveServerId;
public String binlogFileName;
public BinlogDumpCommandPacket(){
setCommand((byte) 0x12);
}
public void fromBytes(byte[] data) {
// bypass
}
/**
* <pre>
* Bytes Name
* ----- ----
* 1 command
* n arg
* --------------------------------------------------------
* Bytes Name
* ----- ----
* 4 binlog position to start at (little endian)
* 2 binlog flags (currently not used; always 0)
* 4 server_id of the slave (little endian)
* n binlog file name (optional)
*
* </pre>
*/
public byte[] toBytes() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
// 0. write command number
out.write(getCommand());
// 1. write 4 bytes bin-log position to start at
ByteHelper.writeUnsignedIntLittleEndian(binlogPosition, out);
// 2. write 2 bytes bin-log flags
int binlog_flags = 0;
binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
out.write(binlog_flags);
out.write(0x00);
// 3. write 4 bytes server id of the slave
ByteHelper.writeUnsignedIntLittleEndian(this.slaveServerId, out);
// 4. write bin-log file name if necessary
if (StringUtils.isNotEmpty(this.binlogFileName)) {
out.write(this.binlogFileName.getBytes());
}
return out.toByteArray();
}
}

130
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java Normal file
View File

@@ -0,0 +1,130 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC;
import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter;
public class ClientAuthenticationPacket extends PacketWithHeaderPacket {
private String username;
private String password;
private byte charsetNumber;
private String databaseName;
private int serverCapabilities;
private byte[] scrumbleBuff;
public void fromBytes(byte[] data) {
// bypass since nowhere to use.
}
/**
* <pre>
* VERSION 4.1
* Bytes Name
* ----- ----
* 4 client_flags
* 4 max_packet_size
* 1 charset_number
* 23 (filler) always 0x00...
* n (Null-Terminated String) user
* n (Length Coded Binary) scramble_buff (1 + x bytes)
* n (Null-Terminated String) databasename (optional)
* </pre>
*
* @throws IOException
*/
public byte[] toBytes() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
// 1. write client_flags
// 1|4|512|1024|8192|32768
/**
* CLIENT_LONG_PASSWORD CLIENT_LONG_FLAG CLIENT_PROTOCOL_41
* CLIENT_INTERACTIVE CLIENT_TRANSACTIONS CLIENT_SECURE_CONNECTION
*/
ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768, out); // remove
// client_interactive
// feature
// 2. write max_packet_size
ByteHelper.writeUnsignedIntLittleEndian(MSC.MAX_PACKET_LENGTH, out);
// 3. write charset_number
out.write(this.charsetNumber);
// 4. write (filler) always 0x00...
out.write(new byte[23]);
// 5. write (Null-Terminated String) user
ByteHelper.writeNullTerminatedString(getUsername(), out);
// 6. write (Length Coded Binary) scramble_buff (1 + x bytes)
if (StringUtils.isEmpty(getPassword())) {
out.write(0x00);
} else {
try {
byte[] encryptedPassword = MySQLPasswordEncrypter.scramble411(getPassword().getBytes(), scrumbleBuff);
ByteHelper.writeBinaryCodedLengthBytes(encryptedPassword, out);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException("can't encrypt password that will be sent to MySQL server.", e);
}
}
// 7 . (Null-Terminated String) databasename (optional)
if (getDatabaseName() != null) {
ByteHelper.writeNullTerminatedString(getDatabaseName(), out);
}
// end write
return out.toByteArray();
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public void setCharsetNumber(byte charsetNumber) {
this.charsetNumber = charsetNumber;
}
public byte getCharsetNumber() {
return charsetNumber;
}
public void setDatabaseName(String databaseName) {
this.databaseName = databaseName;
}
public String getDatabaseName() {
return databaseName;
}
public void setServerCapabilities(int serverCapabilities) {
this.serverCapabilities = serverCapabilities;
}
public int getServerCapabilities() {
return serverCapabilities;
}
public void setScrumbleBuff(byte[] scrumbleBuff) {
this.scrumbleBuff = scrumbleBuff;
}
public byte[] getScrumbleBuff() {
return scrumbleBuff;
}
}

34
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QueryCommandPacket.java Normal file
View File

@@ -0,0 +1,34 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
public class QueryCommandPacket extends CommandPacket {
private String queryString;
public QueryCommandPacket(){
setCommand((byte) 0x03);
}
public void fromBytes(byte[] data) throws IOException {
}
public byte[] toBytes() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
out.write(getCommand());
out.write(getQueryString().getBytes("UTF-8"));// 链接建立时默认指定编码为UTF-8
return out.toByteArray();
}
public void setQueryString(String queryString) {
this.queryString = queryString;
}
public String getQueryString() {
return queryString;
}
}

17
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/DataPacket.java Normal file
View File

@@ -0,0 +1,17 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
public class DataPacket extends CommandPacket {
public void fromBytes(byte[] data) {
}
public byte[] toBytes() throws IOException {
return null;
}
}

46
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/EOFPacket.java Normal file
View File

@@ -0,0 +1,46 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
public class EOFPacket extends PacketWithHeaderPacket {
public byte fieldCount;
public int warningCount;
public int statusFlag;
/**
* <pre>
* VERSION 4.1
* Bytes Name
* ----- ----
* 1 field_count, always = 0xfe
* 2 warning_count
* 2 Status Flags
* </pre>
*/
public void fromBytes(byte[] data) {
int index = 0;
// 1. read field count
fieldCount = data[index];
index++;
// 2. read warning count
this.warningCount = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
// 3. read status flag
this.statusFlag = ByteHelper.readUnsignedShortLittleEndian(data, index);
// end read
}
public byte[] toBytes() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(5);
out.write(this.fieldCount);
ByteHelper.writeUnsignedShortLittleEndian(this.warningCount, out);
ByteHelper.writeUnsignedShortLittleEndian(this.statusFlag, out);
return out.toByteArray();
}
}

66
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ErrorPacket.java Normal file
View File

@@ -0,0 +1,66 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
public class ErrorPacket extends PacketWithHeaderPacket {
public byte fieldCount;
public int errorNumber;
public byte sqlStateMarker;
public byte[] sqlState;
public String message;
/**
* <pre>
* VERSION 4.1
* Bytes Name
* ----- ----
* 1 field_count, always = 0xff
* 2 errno
* 1 (sqlstate marker), always '#'
* 5 sqlstate (5 characters)
* n message
*
* </pre>
*/
public void fromBytes(byte[] data) {
int index = 0;
// 1. read field count
this.fieldCount = data[0];
index++;
// 2. read error no.
this.errorNumber = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
// 3. read marker
this.sqlStateMarker = data[index];
index++;
// 4. read sqlState
this.sqlState = ByteHelper.readFixedLengthBytes(data, index, 5);
index += 5;
// 5. read message
this.message = new String(ByteHelper.readFixedLengthBytes(data, index, data.length - index));
// end read
}
public byte[] toBytes() throws IOException {
return null;
}
@Override
public String toString() {
return "ErrorPacket [errorNumber=" + errorNumber + ", fieldCount=" + fieldCount + ", message=" + message
+ ", sqlState=" + sqlStateToString() + ", sqlStateMarker=" + (char) sqlStateMarker + "]";
}
private String sqlStateToString() {
StringBuilder builder = new StringBuilder(5);
for (byte b : this.sqlState) {
builder.append((char) b);
}
return builder.toString();
}
}

190
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java Normal file
View File

@@ -0,0 +1,190 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader;
public class FieldPacket extends PacketWithHeaderPacket {
private String catalog;
private String db;
private String table;
private String originalTable;
private String name;
private String originalName;
private int character;
private long length;
private byte type;
private int flags;
private byte decimals;
private String definition;
/**
* <pre>
* VERSION 4.1
* Bytes Name
* ----- ----
* n (Length Coded String) catalog
* n (Length Coded String) db
* n (Length Coded String) table
* n (Length Coded String) org_table
* n (Length Coded String) name
* n (Length Coded String) org_name
* 1 (filler)
* 2 charsetnr
* 4 length
* 1 type
* 2 flags
* 1 decimals
* 2 (filler), always 0x00
* n (Length Coded Binary) default
*
* </pre>
*/
public void fromBytes(byte[] data) throws IOException {
int index = 0;
LengthCodedStringReader reader = new LengthCodedStringReader(null, index);
// 1.
catalog = reader.readLengthCodedString(data);
// 2.
db = reader.readLengthCodedString(data);
this.table = reader.readLengthCodedString(data);
this.originalTable = reader.readLengthCodedString(data);
this.name = reader.readLengthCodedString(data);
this.originalName = reader.readLengthCodedString(data);
index = reader.getIndex();
//
index++;
//
this.character = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
//
this.length = ByteHelper.readUnsignedIntLittleEndian(data, index);
index += 4;
//
this.type = data[index];
index++;
//
this.flags = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
//
this.decimals = data[index];
index++;
//
if (index < data.length) {
reader.setIndex(index);
this.definition = reader.readLengthCodedString(data);
}
}
public byte[] toBytes() throws IOException {
return null;
}
public String getCatalog() {
return catalog;
}
public void setCatalog(String catalog) {
this.catalog = catalog;
}
public String getDb() {
return db;
}
public void setDb(String db) {
this.db = db;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getOriginalTable() {
return originalTable;
}
public void setOriginalTable(String originalTable) {
this.originalTable = originalTable;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getOriginalName() {
return originalName;
}
public void setOriginalName(String originalName) {
this.originalName = originalName;
}
public int getCharacter() {
return character;
}
public void setCharacter(int character) {
this.character = character;
}
public long getLength() {
return length;
}
public void setLength(long length) {
this.length = length;
}
public byte getType() {
return type;
}
public void setType(byte type) {
this.type = type;
}
public int getFlags() {
return flags;
}
public void setFlags(int flags) {
this.flags = flags;
}
public byte getDecimals() {
return decimals;
}
public void setDecimals(byte decimals) {
this.decimals = decimals;
}
public String getDefinition() {
return definition;
}
public void setDefinition(String definition) {
this.definition = definition;
}
public String toString() {
return "FieldPacket [catalog=" + catalog + ", character=" + character + ", db=" + db + ", decimals=" + decimals
+ ", definition=" + definition + ", flags=" + flags + ", length=" + length + ", name=" + name
+ ", originalName=" + originalName + ", originalTable=" + originalTable + ", table=" + table + ", type="
+ type + "]";
}
}

91
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/HandshakeInitializationPacket.java Normal file
View File

@@ -0,0 +1,91 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC;
/**
* MySQL Handshake Initialization Packet.<br>
*
* @author fujohnwang
* @since 1.0
*/
public class HandshakeInitializationPacket extends PacketWithHeaderPacket {
public byte protocolVersion = MSC.DEFAULT_PROTOCOL_VERSION;
public String serverVersion;
public long threadId;
public byte[] seed;
public int serverCapabilities;
public byte serverCharsetNumber;
public int serverStatus;
public byte[] restOfScrambleBuff;
public HandshakeInitializationPacket(){
}
public HandshakeInitializationPacket(HeaderPacket header){
super(header);
}
/**
* <pre>
* Bytes Name
* ----- ----
* 1 protocol_version
* n (Null-Terminated String) server_version
* 4 thread_id
* 8 scramble_buff
* 1 (filler) always 0x00
* 2 server_capabilities
* 1 server_language
* 2 server_status
* 13 (filler) always 0x00 ...
* 13 rest of scramble_buff (4.1)
* </pre>
*/
public void fromBytes(byte[] data) {
int index = 0;
// 1. read protocol_version
protocolVersion = data[index];
index++;
// 2. read server_version
byte[] serverVersionBytes = ByteHelper.readNullTerminatedBytes(data, index);
serverVersion = new String(serverVersionBytes);
index += (serverVersionBytes.length + 1);
// 3. read thread_id
threadId = ByteHelper.readUnsignedIntLittleEndian(data, index);
index += 4;
// 4. read scramble_buff
seed = ByteHelper.readFixedLengthBytes(data, index, 8);
index += 8;
index += 1; // 1 byte (filler) always 0x00
// 5. read server_capabilities
this.serverCapabilities = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
// 6. read server_language
this.serverCharsetNumber = data[index];
index++;
// 7. read server_status
this.serverStatus = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
// 8. bypass filtered bytes
index += 13;
// 9. read rest of scramble_buff
this.restOfScrambleBuff = ByteHelper.readFixedLengthBytes(data, index, 12); // 虽然Handshake Initialization
// Packet规定最后13个byte是剩下的scrumble,
// 但实际上最后一个字节是0, 不应该包含在scrumble中.
// end read
}
/**
* Bypass implementing it, 'cause nowhere to use it.
*/
public byte[] toBytes() throws IOException {
return null;
}
}

118
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/OKPacket.java Normal file
View File

@@ -0,0 +1,118 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import java.util.Arrays;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
/**
* Aka. OK packet
*
* @author fujohnwang
*/
public class OKPacket extends PacketWithHeaderPacket {
public byte fieldCount;
public byte[] affectedRows;
public byte[] insertId;
public int serverStatus;
public int warningCount;
public String message;
/**
* <pre>
* VERSION 4.1
* Bytes Name
* ----- ----
* 1 (Length Coded Binary) field_count, always = 0
* 1-9 (Length Coded Binary) affected_rows
* 1-9 (Length Coded Binary) insert_id
* 2 server_status
* 2 warning_count
* n (until end of packet) message
* </pre>
*
* @throws IOException
*/
public void fromBytes(byte[] data) throws IOException {
int index = 0;
// 1. read field count
this.fieldCount = data[0];
index++;
// 2. read affected rows
this.affectedRows = ByteHelper.readBinaryCodedLengthBytes(data, index);
index += this.affectedRows.length;
// 3. read insert id
this.insertId = ByteHelper.readBinaryCodedLengthBytes(data, index);
index += this.insertId.length;
// 4. read server status
this.serverStatus = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
// 5. read warning count
this.warningCount = ByteHelper.readUnsignedShortLittleEndian(data, index);
index += 2;
// 6. read message.
this.message = new String(ByteHelper.readFixedLengthBytes(data, index, data.length - index));
// end read
}
public byte[] toBytes() throws IOException {
return null;
}
public byte getFieldCount() {
return fieldCount;
}
public void setFieldCount(byte fieldCount) {
this.fieldCount = fieldCount;
}
public byte[] getAffectedRows() {
return affectedRows;
}
public void setAffectedRows(byte[] affectedRows) {
this.affectedRows = affectedRows;
}
public byte[] getInsertId() {
return insertId;
}
public void setInsertId(byte[] insertId) {
this.insertId = insertId;
}
public int getServerStatus() {
return serverStatus;
}
public void setServerStatus(int serverStatus) {
this.serverStatus = serverStatus;
}
public int getWarningCount() {
return warningCount;
}
public void setWarningCount(int warningCount) {
this.warningCount = warningCount;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public String toString() {
return "OKPacket [affectedRows=" + Arrays.toString(affectedRows) + ", fieldCount=" + fieldCount + ", insertId="
+ Arrays.toString(insertId) + ", message=" + message + ", serverStatus=" + serverStatus
+ ", warningCount=" + warningCount + "]";
}
}

27
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/Reply323Packet.java Normal file
View File

@@ -0,0 +1,27 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
public class Reply323Packet extends PacketWithHeaderPacket {
public byte[] seed;
public void fromBytes(byte[] data) throws IOException {
}
public byte[] toBytes() throws IOException {
if (seed == null) {
return new byte[] { (byte) 0 };
} else {
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteHelper.writeNullTerminated(seed, out);
return out.toByteArray();
}
}
}

67
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetHeaderPacket.java Normal file
View File

@@ -0,0 +1,67 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
/**
* <pre>
* Type Of Result Packet Hexadecimal Value Of First Byte (field_count)
* ---------------------------------------------------------------------------
* Result Set Packet 1-250 (first byte of Length-Coded Binary)
* </pre>
*
* The sequence of result set packet:
*
* <pre>
* (Result Set Header Packet) the number of columns
* (Field Packets) column descriptors
* (EOF Packet) marker: end of Field Packets
* (Row Data Packets) row contents
* (EOF Packet) marker: end of Data Packets
*
* <pre>
*
* @author fujohnwang
*/
public class ResultSetHeaderPacket extends PacketWithHeaderPacket {
private long columnCount;
private long extra;
public void fromBytes(byte[] data) throws IOException {
int index = 0;
byte[] colCountBytes = ByteHelper.readBinaryCodedLengthBytes(data, index);
columnCount = ByteHelper.readLengthCodedBinary(colCountBytes, index);
index += colCountBytes.length;
if (index < data.length - 1) {
extra = ByteHelper.readLengthCodedBinary(data, index);
}
}
public byte[] toBytes() throws IOException {
return null;
}
public long getColumnCount() {
return columnCount;
}
public void setColumnCount(long columnCount) {
this.columnCount = columnCount;
}
public long getExtra() {
return extra;
}
public void setExtra(long extra) {
this.extra = extra;
}
public String toString() {
return "ResultSetHeaderPacket [columnCount=" + columnCount + ", extra=" + extra + "]";
}
}

42
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java Normal file
View File

@@ -0,0 +1,42 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.List;
public class ResultSetPacket {
private SocketAddress sourceAddress;
private List<FieldPacket> fieldDescriptors = new ArrayList<FieldPacket>();
private List<String> fieldValues = new ArrayList<String>();
public void setFieldDescriptors(List<FieldPacket> fieldDescriptors) {
this.fieldDescriptors = fieldDescriptors;
}
public List<FieldPacket> getFieldDescriptors() {
return fieldDescriptors;
}
public void setFieldValues(List<String> fieldValues) {
this.fieldValues = fieldValues;
}
public List<String> getFieldValues() {
return fieldValues;
}
public void setSourceAddress(SocketAddress sourceAddress) {
this.sourceAddress = sourceAddress;
}
public SocketAddress getSourceAddress() {
return sourceAddress;
}
public String toString() {
return "ResultSetPacket [fieldDescriptors=" + fieldDescriptors + ", fieldValues=" + fieldValues
+ ", sourceAddress=" + sourceAddress + "]";
}
}

38
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java Normal file
View File

@@ -0,0 +1,38 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader;
public class RowDataPacket extends PacketWithHeaderPacket {
private List<String> columns = new ArrayList<String>();
public void fromBytes(byte[] data) throws IOException {
int index = 0;
LengthCodedStringReader reader = new LengthCodedStringReader(null, index);
do {
getColumns().add(reader.readLengthCodedString(data));
} while (reader.getIndex() < data.length);
}
public byte[] toBytes() throws IOException {
return null;
}
public void setColumns(List<String> columns) {
this.columns = columns;
}
public List<String> getColumns() {
return columns;
}
public String toString() {
return "RowDataPacket [columns=" + columns + "]";
}
}

38
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/BinlogDumpCommandBuilder.java Normal file
View File

@@ -0,0 +1,38 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
public class BinlogDumpCommandBuilder {
public BinlogDumpCommandPacket build(String binglogFile, long position, long slaveId) {
BinlogDumpCommandPacket command = new BinlogDumpCommandPacket();
command.binlogPosition = position;
if (!StringUtils.isEmpty(binglogFile)) {
command.binlogFileName = binglogFile;
}
command.slaveServerId = slaveId;
// end settings.
return command;
}
public ChannelBuffer toChannelBuffer(BinlogDumpCommandPacket command) throws IOException {
byte[] commandBytes = command.toBytes();
byte[] headerBytes = assembleHeaderBytes(commandBytes.length);
ChannelBuffer buffer = ChannelBuffers.wrappedBuffer(headerBytes, commandBytes);
return buffer;
}
private byte[] assembleHeaderBytes(int length) {
HeaderPacket header = new HeaderPacket();
header.setPacketBodyLength(length);
header.setPacketSequenceNumber((byte) 0x00);
return header.toBytes();
}
}

158
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java Normal file
View File

@@ -0,0 +1,158 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
public abstract class ByteHelper {
public static final long NULL_LENGTH = -1;
public static byte[] readNullTerminatedBytes(byte[] data, int index) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
for (int i = index; i < data.length; i++) {
byte item = data[i];
if (item == MSC.NULL_TERMINATED_STRING_DELIMITER) {
break;
}
out.write(item);
}
return out.toByteArray();
}
public static void writeNullTerminatedString(String str, ByteArrayOutputStream out) throws IOException {
out.write(str.getBytes());
out.write(MSC.NULL_TERMINATED_STRING_DELIMITER);
}
public static void writeNullTerminated(byte[] data, ByteArrayOutputStream out) throws IOException {
out.write(data);
out.write(MSC.NULL_TERMINATED_STRING_DELIMITER);
}
public static byte[] readFixedLengthBytes(byte[] data, int index, int length) {
byte[] bytes = new byte[length];
System.arraycopy(data, index, bytes, 0, length);
return bytes;
}
/**
* Read 4 bytes in Little-endian byte order.
*
* @param data, the original byte array
* @param index, start to read from.
* @return
*/
public static long readUnsignedIntLittleEndian(byte[] data, int index) {
long result = (long) (data[index] & 0xFF) | (long) ((data[index + 1] & 0xFF) << 8)
| (long) ((data[index + 2] & 0xFF) << 16) | (long) ((data[index + 3] & 0xFF) << 24);
return result;
}
public static long readUnsignedLongLittleEndian(byte[] data, int index) {
long accumulation = 0;
int position = index;
for (int shiftBy = 0; shiftBy < 64; shiftBy += 8) {
accumulation |= (long) ((data[position++] & 0xff) << shiftBy);
}
return accumulation;
}
public static int readUnsignedShortLittleEndian(byte[] data, int index) {
int result = (data[index] & 0xFF) | ((data[index + 1] & 0xFF) << 8);
return result;
}
public static int readUnsignedMediumLittleEndian(byte[] data, int index) {
int result = (data[index] & 0xFF) | ((data[index + 1] & 0xFF) << 8) | ((data[index + 2] & 0xFF) << 16);
return result;
}
public static long readLengthCodedBinary(byte[] data, int index) throws IOException {
int firstByte = data[index] & 0xFF;
switch (firstByte) {
case 251:
return NULL_LENGTH;
case 252:
return readUnsignedShortLittleEndian(data, index + 1);
case 253:
return readUnsignedMediumLittleEndian(data, index + 1);
case 254:
return readUnsignedLongLittleEndian(data, index + 1);
default:
return firstByte;
}
}
public static byte[] readBinaryCodedLengthBytes(byte[] data, int index) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
out.write(data[index]);
byte[] buffer = null;
int value = data[index] & 0xFF;
if (value == 251) {
buffer = new byte[0];
}
if (value == 252) {
buffer = new byte[2];
}
if (value == 253) {
buffer = new byte[3];
}
if (value == 254) {
buffer = new byte[8];
}
if (buffer != null) {
System.arraycopy(data, index + 1, buffer, 0, buffer.length);
out.write(buffer);
}
return out.toByteArray();
}
public static void writeUnsignedIntLittleEndian(long data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) (data >>> 8));
out.write((byte) (data >>> 16));
out.write((byte) (data >>> 24));
}
public static void writeUnsignedShortLittleEndian(int data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) ((data >>> 8) & 0xFF));
}
public static void writeUnsignedMediumLittleEndian(int data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) ((data >>> 8) & 0xFF));
out.write((byte) ((data >>> 16) & 0xFF));
}
public static void writeBinaryCodedLengthBytes(byte[] data, ByteArrayOutputStream out) throws IOException {
// 1. write length byte/bytes
if (data.length < 252) {
out.write((byte) data.length);
} else if (data.length < (1 << 16L)) {
out.write((byte) 252);
writeUnsignedShortLittleEndian(data.length, out);
} else if (data.length < (1 << 24L)) {
out.write((byte) 253);
writeUnsignedMediumLittleEndian(data.length, out);
} else {
out.write((byte) 254);
writeUnsignedIntLittleEndian(data.length, out);
}
// 2. write real data followed length byte/bytes
out.write(data);
}
public static void writeFixedLengthBytes(byte[] data, int index, int length, ByteArrayOutputStream out) {
for (int i = index; i < index + length; i++) {
out.write(data[i]);
}
}
public static void writeFixedLengthBytesFromStart(byte[] data, int length, ByteArrayOutputStream out) {
writeFixedLengthBytes(data, 0, length, out);
}
}

62
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ChannelBufferHelper.java Normal file
View File

@@ -0,0 +1,62 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.io.IOException;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.IPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
public class ChannelBufferHelper {
protected transient final Logger logger = LoggerFactory.getLogger(ChannelBufferHelper.class);
public final HeaderPacket assembleHeaderPacket(ChannelBuffer buffer) {
HeaderPacket header = new HeaderPacket();
byte[] headerBytes = new byte[MSC.HEADER_PACKET_LENGTH];
buffer.readBytes(headerBytes);
header.fromBytes(headerBytes);
return header;
}
public final PacketWithHeaderPacket assembleBodyPacketWithHeader(ChannelBuffer buffer, HeaderPacket header,
PacketWithHeaderPacket body) throws IOException {
if (body.getHeader() == null) {
body.setHeader(header);
}
logger.debug("body packet type:{}", body.getClass());
logger.debug("read body packet with packet length: {} ", header.getPacketBodyLength());
byte[] packetBytes = new byte[header.getPacketBodyLength()];
logger.debug("readable bytes before reading body:{}", buffer.readableBytes());
buffer.readBytes(packetBytes);
body.fromBytes(packetBytes);
logger.debug("body packet: {}", body);
return body;
}
public final ChannelBuffer createHeaderWithPacketNumberPlusOne(int bodyLength, byte packetNumber) {
HeaderPacket header = new HeaderPacket();
header.setPacketBodyLength(bodyLength);
header.setPacketSequenceNumber((byte) (packetNumber + 1));
return ChannelBuffers.wrappedBuffer(header.toBytes());
}
public final ChannelBuffer createHeader(int bodyLength, byte packetNumber) {
HeaderPacket header = new HeaderPacket();
header.setPacketBodyLength(bodyLength);
header.setPacketSequenceNumber(packetNumber);
return ChannelBuffers.wrappedBuffer(header.toBytes());
}
public final ChannelBuffer buildChannelBufferFromCommandPacket(IPacket packet) throws IOException {
byte[] bodyBytes = packet.toBytes();
ChannelBuffer header = createHeader(bodyBytes.length, (byte) 0);
return ChannelBuffers.wrappedBuffer(header, ChannelBuffers.wrappedBuffer(bodyBytes));
}
}

43
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/LengthCodedStringReader.java Normal file
View File

@@ -0,0 +1,43 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.io.IOException;
import org.apache.commons.lang.ArrayUtils;
public class LengthCodedStringReader {
public static final String CODE_PAGE_1252 = "UTF-8";
private String encoding;
private int index = 0; // 数组下标
public LengthCodedStringReader(String encoding, int startIndex){
this.encoding = encoding;
this.index = startIndex;
}
public String readLengthCodedString(byte[] data) throws IOException {
byte[] lengthBytes = ByteHelper.readBinaryCodedLengthBytes(data, getIndex());
long length = ByteHelper.readLengthCodedBinary(data, getIndex());
setIndex(getIndex() + lengthBytes.length);
if (ByteHelper.NULL_LENGTH == length) {
return null;
}
try {
return new String(ArrayUtils.subarray(data, getIndex(), (int) (getIndex() + length)),
encoding == null ? CODE_PAGE_1252 : encoding);
} finally {
setIndex((int) (getIndex() + length));
}
}
public void setIndex(int index) {
this.index = index;
}
public int getIndex() {
return index;
}
}

26
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MSC.java Normal file
View File

@@ -0,0 +1,26 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
/**
* MySQL Constants.<br>
* constants that is used in mysql server.<br>
*
* @author fujohnwang
*/
public abstract class MSC {
public static final int MAX_PACKET_LENGTH = (1 << 24);
public static final int HEADER_PACKET_LENGTH_FIELD_LENGTH = 3;
public static final int HEADER_PACKET_LENGTH_FIELD_OFFSET = 0;
public static final int HEADER_PACKET_LENGTH = 4;
public static final int HEADER_PACKET_NUMBER_FIELD_LENGTH = 1;
public static final byte NULL_TERMINATED_STRING_DELIMITER = 0x00;
public static final byte DEFAULT_PROTOCOL_VERSION = 0x0a;
public static final int FIELD_COUNT_FIELD_LENGTH = 1;
public static final int EVENT_TYPE_OFFSET = 4;
public static final int EVENT_LEN_OFFSET = 9;
public static final long DEFAULT_BINLOG_FILE_START_POSITION = 4;
}

74
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MySQLPasswordEncrypter.java Normal file
View File

@@ -0,0 +1,74 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MySQLPasswordEncrypter {
public static final byte[] scramble411(byte[] pass, byte[] seed) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("SHA-1");
byte[] pass1 = md.digest(pass);
md.reset();
byte[] pass2 = md.digest(pass1);
md.reset();
md.update(seed);
byte[] pass3 = md.digest(pass2);
for (int i = 0; i < pass3.length; i++) {
pass3[i] = (byte) (pass3[i] ^ pass1[i]);
}
return pass3;
}
public static String scramble323(String pass, String seed) {
if ((pass == null) || (pass.length() == 0)) {
return pass;
}
byte b;
double d;
long[] pw = hash(seed);
long[] msg = hash(pass);
long max = 0x3fffffffL;
long seed1 = (pw[0] ^ msg[0]) % max;
long seed2 = (pw[1] ^ msg[1]) % max;
char[] chars = new char[seed.length()];
for (int i = 0; i < seed.length(); i++) {
seed1 = ((seed1 * 3) + seed2) % max;
seed2 = (seed1 + seed2 + 33) % max;
d = (double) seed1 / (double) max;
b = (byte) java.lang.Math.floor((d * 31) + 64);
chars[i] = (char) b;
}
seed1 = ((seed1 * 3) + seed2) % max;
seed2 = (seed1 + seed2 + 33) % max;
d = (double) seed1 / (double) max;
b = (byte) java.lang.Math.floor(d * 31);
for (int i = 0; i < seed.length(); i++) {
chars[i] ^= (char) b;
}
return new String(chars);
}
private static long[] hash(String src) {
long nr = 1345345333L;
long add = 7;
long nr2 = 0x12345671L;
long tmp;
for (int i = 0; i < src.length(); ++i) {
switch (src.charAt(i)) {
case ' ':
case '\t':
continue;
default:
tmp = (0xff & src.charAt(i));
nr ^= ((((nr & 63) + add) * tmp) + (nr << 8));
nr2 += ((nr2 << 8) ^ nr);
add += tmp;
}
}
long[] result = new long[2];
result[0] = nr & 0x7fffffffL;
result[1] = nr2 & 0x7fffffffL;
return result;
}
}

67
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java Normal file
View File

@@ -0,0 +1,67 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
public abstract class PacketManager {
public static HeaderPacket readHeader(SocketChannel ch, int len) throws IOException {
HeaderPacket header = new HeaderPacket();
header.fromBytes(readBytesAsBuffer(ch, len).array());
return header;
}
public static ByteBuffer readBytesAsBuffer(SocketChannel ch, int len) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(len);
while (buffer.hasRemaining()) {
int readNum = ch.read(buffer);
if (readNum == -1) {
throw new IOException("Unexpected End Stream");
}
}
return buffer;
}
public static byte[] readBytes(SocketChannel ch, int len) throws IOException {
return readBytesAsBuffer(ch, len).array();
}
/**
* Since We r using blocking IO, so we will just write once and assert the length to simplify the read operation.<br>
* If the block write doesn't work as we expected, we will change this implementation as per the result.
*
* @param ch
* @param len
* @return
* @throws IOException
*/
public static void write(SocketChannel ch, ByteBuffer[] srcs) throws IOException {
@SuppressWarnings("unused")
long total = 0;
for (ByteBuffer buffer : srcs) {
total += buffer.remaining();
}
ch.write(srcs);
// https://github.com/alibaba/canal/issues/24
// 部分windows用户会出现size != total的情况，jdk为java7/openjdk，估计和java版本有关，暂时不做检查
// long size = ch.write(srcs);
// if (size != total) {
// throw new IOException("unexpected blocking io behavior");
// }
}
public static void write(SocketChannel ch, byte[] body) throws IOException {
write(ch, body, (byte) 0);
}
public static void write(SocketChannel ch, byte[] body, byte packetSeqNumber) throws IOException {
HeaderPacket header = new HeaderPacket();
header.setPacketBodyLength(body.length);
header.setPacketSequenceNumber(packetSeqNumber);
write(ch, new ByteBuffer[] { ByteBuffer.wrap(header.toBytes()), ByteBuffer.wrap(body) });
}
}

54
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java Normal file
View File

@@ -0,0 +1,54 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import java.io.IOException;
import java.net.InetSocketAddress;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
public class MysqlConnectorTest {
@Test
public void testQuery() {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
try {
connector.connect();
MysqlQueryExecutor executor = new MysqlQueryExecutor(connector);
ResultSetPacket result = executor.query("show variables like '%char%';");
System.out.println(result);
result = executor.query("select * from test.test1");
System.out.println(result);
} catch (IOException e) {
Assert.fail(e.getMessage());
} finally {
try {
connector.disconnect();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
// @Test
public void testUpdate() {
MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
try {
connector.connect();
MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector);
executor.update("insert into test.test2(id,name,score,text_value) values(null,'中文1',10,'中文2')");
} catch (IOException e) {
Assert.fail(e.getMessage());
} finally {
try {
connector.disconnect();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
}