This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,114 @@
package com.alibaba.otter.canal.protocol;
option java_package = "com.alibaba.otter.canal.protocol";
option java_outer_classname = "CanalPacket";
option optimize_for = SPEED;
enum Compression {
NONE = 1;
ZLIB = 2;
GZIP = 3;
LZF = 4;
}
enum PacketType {
HANDSHAKE = 1;
CLIENTAUTHENTICATION = 2;
ACK = 3;
SUBSCRIPTION = 4;
UNSUBSCRIPTION = 5;
GET = 6;
MESSAGES = 7;
CLIENTACK = 8;
// management part
SHUTDOWN = 9;
// integration
DUMP = 10;
HEARTBEAT = 11;
CLIENTROLLBACK = 12;
}
message Packet {
optional int32 magic_number = 1 [default = 17];
optional int32 version = 2 [default = 1];
optional PacketType type = 3;
optional Compression compression = 4 [default = NONE];
optional bytes body = 5;
}
message HeartBeat {
optional int64 send_timestamp = 1;
optional int64 start_timestamp = 2;
}
message Handshake {
optional string communication_encoding = 1 [default = "utf8"];
optional bytes seeds = 2;
repeated Compression supported_compressions = 3;
}
// client authentication
message ClientAuth {
optional string username = 1;
optional bytes password = 2; // hashed password with seeds from Handshake message
optional int32 net_read_timeout = 3 [default = 0]; // in seconds
optional int32 net_write_timeout = 4 [default = 0]; // in seconds
optional string destination = 5;
optional string client_id = 6;
optional string filter = 7;
optional int64 start_timestamp = 8;
}
message Ack {
optional int32 error_code = 1 [default = 0];
optional string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
}
message ClientAck {
optional string destination = 1;
optional string client_id = 2;
optional int64 batch_id = 3;
}
// subscription
message Sub {
optional string destination = 1;
optional string client_id = 2;
optional string filter = 7;
}
// Unsubscription
message Unsub {
optional string destination = 1;
optional string client_id = 2;
optional string filter = 7;
}
// PullRequest
message Get {
optional string destination = 1;
optional string client_id = 2;
optional int32 fetch_size = 3;
optional int64 timeout = 4 [default = -1]; // 默认-1时代表不控制
optional int32 unit = 5 [default = 2];// 数字类型,0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
optional bool auto_ack = 6 [default = false]; // 是否自动ack
}
//
message Messages {
optional int64 batch_id = 1;
repeated bytes messages = 2;
}
// TBD when new packets are required
message Dump{
optional string journal = 1;
optional int64 position = 2;
optional int64 timestamp = 3 [default = 0];
}
message ClientRollback{
optional string destination = 1;
optional string client_id = 2;
optional int64 batch_id = 3;
}

View File

@@ -0,0 +1,102 @@
package com.alibaba.otter.canal.protocol;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* @author zebin.xuzb @ 2012-6-20
* @version 1.0.0
*/
public class ClientIdentity {
private String destination;
private short clientId;
private String filter;
public ClientIdentity(){
}
public ClientIdentity(String destination, short clientId){
this.clientId = clientId;
this.destination = destination;
}
public ClientIdentity(String destination, short clientId, String filter){
this.clientId = clientId;
this.destination = destination;
this.filter = filter;
}
public Boolean hasFilter() {
if (filter == null) {
return false;
}
return StringUtils.isNotBlank(filter);
}
// ======== setter =========
public String getDestination() {
return destination;
}
public short getClientId() {
return clientId;
}
public void setClientId(short clientId) {
this.clientId = clientId;
}
public void setDestination(String destination) {
this.destination = destination;
}
public String getFilter() {
return filter;
}
public void setFilter(String filter) {
this.filter = filter;
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + clientId;
result = prime * result + ((destination == null) ? 0 : destination.hashCode());
return result;
}
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof ClientIdentity)) {
return false;
}
ClientIdentity other = (ClientIdentity) obj;
if (clientId != other.clientId) {
return false;
}
if (destination == null) {
if (other.destination != null) {
return false;
}
} else if (!destination.equals(other.destination)) {
return false;
}
return true;
}
}

View File

@@ -0,0 +1,196 @@
package com.alibaba.otter.canal.protocol;
option java_package = "com.alibaba.otter.canal.protocol";
option java_outer_classname = "CanalEntry";
option optimize_for = SPEED;
/****************************************************************
* message model
*如果要在Enum中新增类型,确保以前的类型的下标值不变.
****************************************************************/
message Entry {
/**协议头部信息**/
optional Header header = 1;
/**打散后的事件类型**/
optional EntryType entryType = 2 [default = ROWDATA];
/**传输的二进制数组**/
optional bytes storeValue = 3;
}
/**message Header**/
message Header {
/**协议的版本号**/
optional int32 version = 1 [default = 1];
/**binlog/redolog 文件名**/
optional string logfileName = 2;
/**binlog/redolog 文件的偏移位置**/
optional int64 logfileOffset = 3;
/**服务端serverId**/
optional int64 serverId = 4;
/** 变更数据的编码 **/
optional string serverenCode = 5;
/**变更数据的执行时间 **/
optional int64 executeTime = 6;
/** 变更数据的来源**/
optional Type sourceType = 7 [default = MYSQL];
/** 变更数据的schemaname**/
optional string schemaName = 8;
/**变更数据的tablename**/
optional string tableName = 9;
/**每个event的长度**/
optional int64 eventLength = 10;
/**数据变更类型**/
optional EventType eventType = 11 [default = UPDATE];
/**预留扩展**/
repeated Pair props = 12;
}
/**每个字段的数据结构**/
message Column {
/**字段下标**/
optional int32 index = 1;
/**字段java中类型**/
optional int32 sqlType = 2;
/**字段名称(忽略大小写),在mysql中是没有的**/
optional string name = 3;
/**是否是主键**/
optional bool isKey = 4;
/**如果EventType=UPDATE,用于标识这个字段值是否有修改**/
optional bool updated = 5;
/** 标识是否为空 **/
optional bool isNull = 6 [default = false];
/**预留扩展**/
repeated Pair props = 7;
/** 字段值,timestamp,Datetime是一个时间格式的文本 **/
optional string value = 8;
/** 对应数据对象原始长度 **/
optional int32 length = 9;
/**字段mysql类型**/
optional string mysqlType = 10;
}
message RowData {
/** 字段信息,增量数据(修改前,删除前) **/
repeated Column beforeColumns = 1;
/** 字段信息,增量数据(修改后,新增后) **/
repeated Column afterColumns = 2;
/**预留扩展**/
repeated Pair props = 3;
}
/**message row 每行变更数据的数据结构**/
message RowChange {
/**tableId,由数据库产生**/
optional int64 tableId = 1;
/**数据变更类型**/
optional EventType eventType = 2 [default = UPDATE];
/** 标识是否是ddl语句 **/
optional bool isDdl = 10 [default = false];
/** ddl/query的sql语句 **/
optional string sql = 11;
/** 一次数据库变更可能存在多行 **/
repeated RowData rowDatas = 12;
/**预留扩展**/
repeated Pair props = 13;
/** ddl/query的schemaName,会存在跨库ddl,需要保留执行ddl的当前schemaName **/
optional string ddlSchemaName = 14;
}
/**开始事务的一些信息**/
message TransactionBegin{
/**已废弃,请使用header里的executeTime**/
optional int64 executeTime = 1;
/**已废弃,Begin里不提供事务id**/
optional string transactionId = 2;
/**预留扩展**/
repeated Pair props = 3;
/**执行的thread Id**/
optional int64 threadId = 4;
}
/**结束事务的一些信息**/
message TransactionEnd{
/**已废弃,请使用header里的executeTime**/
optional int64 executeTime = 1;
/**事务号**/
optional string transactionId = 2;
/**预留扩展**/
repeated Pair props = 3;
}
/**预留扩展**/
message Pair{
optional string key = 1;
optional string value = 2;
}
/**打散后的事件类型,主要用于标识事务的开始,变更数据,结束**/
enum EntryType{
TRANSACTIONBEGIN = 1;
ROWDATA = 2;
TRANSACTIONEND = 3;
/** 心跳类型,内部使用,外部暂不可见,可忽略 **/
HEARTBEAT = 4;
}
/** 事件类型 **/
enum EventType {
INSERT = 1;
UPDATE = 2;
DELETE = 3;
CREATE = 4;
ALTER = 5;
ERASE = 6;
QUERY = 7;
TRUNCATE = 8;
RENAME = 9;
/**CREATE INDEX**/
CINDEX = 10;
DINDEX = 11;
}
/**数据库类型**/
enum Type {
ORACLE = 1;
MYSQL = 2;
PGSQL = 3;
}

View File

@@ -0,0 +1,56 @@
package com.alibaba.otter.canal.protocol;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
/**
* @author zebin.xuzb @ 2012-6-19
* @version 1.0.0
*/
public class Message implements Serializable {
private static final long serialVersionUID = 1234034768477580009L;
private long id;
private List<CanalEntry.Entry> entries = new ArrayList<CanalEntry.Entry>();
public Message(long id, List<Entry> entries){
this.id = id;
this.entries = entries == null ? new ArrayList<Entry>() : entries;
}
public Message(long id){
this.id = id;
}
public long getId() {
return id;
}
public void setId(long id) {
this.id = id;
}
public List<Entry> getEntries() {
return entries;
}
public void setEntries(List<CanalEntry.Entry> entries) {
this.entries = entries;
}
public void addEntry(CanalEntry.Entry entry) {
this.entries.add(entry);
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

View File

@@ -0,0 +1,32 @@
package com.alibaba.otter.canal.protocol.exception;
import org.apache.commons.lang.exception.NestableRuntimeException;
/**
* @author zebin.xuzb @ 2012-6-20
* @version 1.0.0
*/
public class CanalClientException extends NestableRuntimeException {
private static final long serialVersionUID = -7545341502620139031L;
public CanalClientException(String errorCode){
super(errorCode);
}
public CanalClientException(String errorCode, Throwable cause){
super(errorCode, cause);
}
public CanalClientException(String errorCode, String errorDesc){
super(errorCode + ":" + errorDesc);
}
public CanalClientException(String errorCode, String errorDesc, Throwable cause){
super(errorCode + ":" + errorDesc, cause);
}
public CanalClientException(Throwable cause){
super(cause);
}
}

View File

@@ -0,0 +1,109 @@
package com.alibaba.otter.canal.protocol.position;
/**
* 数据库对象的唯一标示
*
* @author jianghang 2012-6-14 下午09:20:07
* @version 1.0.0
*/
public class EntryPosition extends TimePosition {
private static final long serialVersionUID = 81432665066427482L;
public static final int EVENTIDENTITY_SEGMENT = 3;
public static final char EVENTIDENTITY_SPLIT = (char) 5;
private boolean included = false;
private String journalName;
private Long position;
public EntryPosition(){
super(null);
}
public EntryPosition(Long timestamp){
this(null, null, timestamp);
}
public EntryPosition(String journalName, Long position){
this(journalName, position, null);
}
public EntryPosition(String journalName, Long position, Long timestamp){
super(timestamp);
this.journalName = journalName;
this.position = position;
}
public String getJournalName() {
return journalName;
}
public void setJournalName(String journalName) {
this.journalName = journalName;
}
public Long getPosition() {
return position;
}
public void setPosition(Long position) {
this.position = position;
}
public boolean isIncluded() {
return included;
}
public void setIncluded(boolean included) {
this.included = included;
}
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + ((journalName == null) ? 0 : journalName.hashCode());
result = prime * result + ((position == null) ? 0 : position.hashCode());
// 手写equals,自动生成时需注意
result = prime * result + ((timestamp == null) ? 0 : timestamp.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!super.equals(obj)) {
return false;
}
if (!(obj instanceof EntryPosition)) {
return false;
}
EntryPosition other = (EntryPosition) obj;
if (journalName == null) {
if (other.journalName != null) {
return false;
}
} else if (!journalName.equals(other.journalName)) {
return false;
}
if (position == null) {
if (other.position != null) {
return false;
}
} else if (!position.equals(other.position)) {
return false;
}
// 手写equals,自动生成时需注意
if (timestamp == null) {
if (other.timestamp != null) {
return false;
}
} else if (!timestamp.equals(other.timestamp)) {
return false;
}
return true;
}
}

View File

@@ -0,0 +1,72 @@
package com.alibaba.otter.canal.protocol.position;
import java.net.InetSocketAddress;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* @author jianghang 2012-6-21 上午10:52:02
* @version 1.0.0
*/
public class LogIdentity extends Position {
private static final long serialVersionUID = 5530225131455662581L;
private InetSocketAddress sourceAddress; // 链接服务器的地址
private Long slaveId; // 对应的slaveId
public LogIdentity(){
}
public LogIdentity(InetSocketAddress sourceAddress, Long slaveId){
this.sourceAddress = sourceAddress;
this.slaveId = slaveId;
}
public InetSocketAddress getSourceAddress() {
return sourceAddress;
}
public void setSourceAddress(InetSocketAddress sourceAddress) {
this.sourceAddress = sourceAddress;
}
public Long getSlaveId() {
return slaveId;
}
public void setSlaveId(Long slaveId) {
this.slaveId = slaveId;
}
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((slaveId == null) ? 0 : slaveId.hashCode());
result = prime * result + ((sourceAddress == null) ? 0 : sourceAddress.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
LogIdentity other = (LogIdentity) obj;
if (slaveId == null) {
if (other.slaveId != null) return false;
} else if (slaveId.longValue() != (other.slaveId.longValue())) return false;
if (sourceAddress == null) {
if (other.sourceAddress != null) return false;
} else if (!sourceAddress.equals(other.sourceAddress)) return false;
return true;
}
}

View File

@@ -0,0 +1,69 @@
package com.alibaba.otter.canal.protocol.position;
/**
* 基于mysql/oracle log位置标示
*
* @author jianghang 2012-6-21 上午10:52:41
* @version 1.0.0
*/
public class LogPosition extends Position {
private static final long serialVersionUID = 3875012010277005819L;
private LogIdentity identity;
private EntryPosition postion;
public LogIdentity getIdentity() {
return identity;
}
public void setIdentity(LogIdentity identity) {
this.identity = identity;
}
public EntryPosition getPostion() {
return postion;
}
public void setPostion(EntryPosition postion) {
this.postion = postion;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((identity == null) ? 0 : identity.hashCode());
result = prime * result + ((postion == null) ? 0 : postion.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof LogPosition)) {
return false;
}
LogPosition other = (LogPosition) obj;
if (identity == null) {
if (other.identity != null) {
return false;
}
} else if (!identity.equals(other.identity)) {
return false;
}
if (postion == null) {
if (other.postion != null) {
return false;
}
} else if (!postion.equals(other.postion)) {
return false;
}
return true;
}
}

View File

@@ -0,0 +1,46 @@
package com.alibaba.otter.canal.protocol.position;
/**
* @author zebin.xuzb 2012-11-3 上午12:23:01
* @since 1.0.0
*/
public class MetaqPosition extends Position {
private static final long serialVersionUID = -8673508769040569273L;
private String topic;
private String msgNewId;
private long offset;
public MetaqPosition(String topic, String msgNewId, long offset){
super();
this.topic = topic;
this.msgNewId = msgNewId;
this.offset = offset;
}
public String getTopic() {
return topic;
}
public String getMsgNewId() {
return msgNewId;
}
public void setTopic(String topic) {
this.topic = topic;
}
public void setMsgNewId(String msgNewId) {
this.msgNewId = msgNewId;
}
public long getOffset() {
return offset;
}
public void setOffset(long offset) {
this.offset = offset;
}
}

View File

@@ -0,0 +1,20 @@
package com.alibaba.otter.canal.protocol.position;
import java.io.Serializable;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* 事件唯一标示
*/
public abstract class Position implements Serializable {
private static final long serialVersionUID = 2332798099928474975L;
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

View File

@@ -0,0 +1,106 @@
package com.alibaba.otter.canal.protocol.position;
import java.io.Serializable;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* 描述一个position范围
*
* @author jianghang 2012-7-10 下午05:28:38
* @version 1.0.0
*/
public class PositionRange<T extends Position> implements Serializable {
private static final long serialVersionUID = -9162037079815694784L;
private T start;
// add by ljh at 2012-09-05,用于记录一个可被ack的位置,保证每次提交到cursor中的位置是一个完整事务的结束
private T ack;
private T end;
public PositionRange(){
}
public PositionRange(T start, T end){
this.start = start;
this.end = end;
}
public T getStart() {
return start;
}
public void setStart(T start) {
this.start = start;
}
public T getEnd() {
return end;
}
public void setEnd(T end) {
this.end = end;
}
public T getAck() {
return ack;
}
public void setAck(T ack) {
this.ack = ack;
}
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((ack == null) ? 0 : ack.hashCode());
result = prime * result + ((end == null) ? 0 : end.hashCode());
result = prime * result + ((start == null) ? 0 : start.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof PositionRange)) {
return false;
}
PositionRange other = (PositionRange) obj;
if (ack == null) {
if (other.ack != null) {
return false;
}
} else if (!ack.equals(other.ack)) {
return false;
}
if (end == null) {
if (other.end != null) {
return false;
}
} else if (!end.equals(other.end)) {
return false;
}
if (start == null) {
if (other.start != null) {
return false;
}
} else if (!start.equals(other.start)) {
return false;
}
return true;
}
}

View File

@@ -0,0 +1,56 @@
package com.alibaba.otter.canal.protocol.position;
/**
* 基于时间的位置,position数据不唯一
*
* @author jianghang 2012-6-14 下午09:22:04
* @version 1.0.0
*/
public class TimePosition extends Position {
private static final long serialVersionUID = 6185261261064226380L;
protected Long timestamp;
public TimePosition(Long timestamp){
this.timestamp = timestamp;
}
public Long getTimestamp() {
return timestamp;
}
public void setTimestamp(Long timestamp) {
this.timestamp = timestamp;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((timestamp == null) ? 0 : timestamp.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof TimePosition)) {
return false;
}
TimePosition other = (TimePosition) obj;
if (timestamp == null) {
if (other.timestamp != null) {
return false;
}
} else if (!timestamp.equals(other.timestamp)) {
return false;
}
return true;
}
}