chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

支持通过GTID同步binlog

Browse Source
This commit is contained in:
hiwjd
2018-05-02 15:28:11 +08:00
parent f46133d116
commit 21f3f8c973
22 changed files with 13479 additions and 12615 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/GTIDSet.java Normal file
View File

@@ -0,0 +1,24 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import java.io.IOException;
/**
* Created by hiwjd on 2018/4/23.
* hiwjd0@gmail.com
*/
public interface GTIDSet {
/**
* 序列化成字节数组
*
* @return
*/
byte[] encode() throws IOException;
/**
* 更新当前实例
* @param str
* @throws Exception
*/
void update(String str);
}

151
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java Normal file
View File

@@ -0,0 +1,151 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.*;
/**
* Created by hiwjd on 2018/4/23.
* hiwjd0@gmail.com
*/
public class MysqlGTIDSet implements GTIDSet {
public Map<String, UUIDSet> sets;
@Override
public byte[] encode() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteHelper.writeUnsignedInt64LittleEndian(sets.size(), out);
for (Map.Entry<String, UUIDSet> entry : sets.entrySet()) {
out.write(entry.getValue().encode());
}
return out.toByteArray();
}
@Override
public void update(String str) {
UUIDSet us = UUIDSet.parse(str);
String sid = us.SID.toString();
if (sets.containsKey(sid)) {
sets.get(sid).intervals.addAll(us.intervals);
sets.get(sid).intervals = UUIDSet.combine(sets.get(sid).intervals);
} else {
sets.put(sid, us);
}
}
@Override
public boolean equals(Object o) {
if (o == null) return false;
if (this == o) return true;
MysqlGTIDSet gs = (MysqlGTIDSet) o;
if (gs.sets == null) return false;
for (Map.Entry<String, UUIDSet> entry : sets.entrySet()) {
if (!entry.getValue().equals(gs.sets.get(entry.getKey()))) {
return false;
}
}
return true;
}
/**
* 解析如下格式的字符串为MysqlGTIDSet:
*
* 726757ad-4455-11e8-ae04-0242ac110002:1 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:2}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:4}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:4 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:5}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:4}, {start:7, stop: 10}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3,726757ad-4455-11e8-ae04-0242ac110003:4 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:4}]
* },
* 726757ad-4455-11e8-ae04-0242ac110003: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:4, stop:5}]
* }
* }
* }
*
* @param gtidData
* @return
*/
public static MysqlGTIDSet parse(String gtidData) {
Map<String, UUIDSet> m;
if (gtidData == null || gtidData.length() < 1) {
m = new HashMap<String, UUIDSet>();
} else {
String[] uuidStrs = gtidData.split(",");
m = new HashMap<String, UUIDSet>(uuidStrs.length);
for (int i = 0; i < uuidStrs.length; i++) {
UUIDSet uuidSet = UUIDSet.parse(uuidStrs[i]);
m.put(uuidSet.SID.toString(), uuidSet);
}
}
MysqlGTIDSet gs = new MysqlGTIDSet();
gs.sets = m;
return gs;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, UUIDSet> entry : sets.entrySet()) {
if (sb.length() > 0) {
sb.append(",");
}
sb.append(entry.getValue().toString());
}
return sb.toString();
}
}

202
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java Normal file
View File

@@ -0,0 +1,202 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
/**
* Created by hiwjd on 2018/4/23.
* hiwjd0@gmail.com
*/
public class UUIDSet {
public UUID SID;
public List<Interval> intervals;
public byte[] encode() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteBuffer bb = ByteBuffer.wrap(new byte[16]);
bb.putLong(SID.getMostSignificantBits());
bb.putLong(SID.getLeastSignificantBits());
out.write(bb.array());
ByteHelper.writeUnsignedInt64LittleEndian(intervals.size(), out);
for (Interval interval : intervals) {
ByteHelper.writeUnsignedInt64LittleEndian(interval.start, out);
ByteHelper.writeUnsignedInt64LittleEndian(interval.stop, out);
}
return out.toByteArray();
}
@Override
public boolean equals(Object o) {
if (o == null) return false;
if (this == o) return true;
UUIDSet us = (UUIDSet) o;
Collections.sort(intervals);
Collections.sort(us.intervals);
if (SID.equals(us.SID) && intervals.equals(us.intervals)) {
return true;
}
return false;
}
public static class Interval implements Comparable<Interval> {
public long start;
public long stop;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Interval interval = (Interval) o;
if (start != interval.start) return false;
return stop == interval.stop;
}
@Override
public int hashCode() {
int result = (int) (start ^ (start >>> 32));
result = 31 * result + (int) (stop ^ (stop >>> 32));
return result;
}
@Override
public int compareTo(Interval o) {
if (equals(o)) {
return 1;
}
return start > o.start ? 1 : (start == o.start ? 0 : -1);
}
}
/**
* 解析如下格式字符串为UUIDSet:
*
* 726757ad-4455-11e8-ae04-0242ac110002:1 =>
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:2}]}
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3 =>
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}]}
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:4
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:5}]}
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}, {start:7, stop:10}]}
*
* @param str
* @return
*/
public static UUIDSet parse(String str) {
String[] ss = str.split(":");
if (ss.length < 2) {
throw new RuntimeException(String.format("parseUUIDSet failed due to wrong format: %s", str));
}
List<Interval> intervals = new ArrayList<Interval>();
for (int i=1; i<ss.length; i++) {
intervals.add(parseInterval(ss[i]));
}
UUIDSet uuidSet = new UUIDSet();
uuidSet.SID = UUID.fromString(ss[0]);
uuidSet.intervals = combine(intervals);
return uuidSet;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(SID.toString());
for (Interval interval : intervals) {
if (interval.start == interval.stop - 1) {
sb.append(":");
sb.append(interval.start);
} else {
sb.append(":");
sb.append(interval.start);
sb.append("-");
sb.append(interval.stop-1);
}
}
return sb.toString();
}
/**
* 解析如下格式字符串为Interval:
*
* 1 => Interval{start:1, stop:2}
* 1-3 => Interval{start:1, stop:4}
*
* 注意！字符串格式表达时[n,m]是两侧都包含的，Interval表达时[n,m)右侧开
*
* @param str
* @return
*/
public static Interval parseInterval(String str) {
String[] ss = str.split("-");
Interval interval = new Interval();
switch (ss.length) {
case 1:
interval.start = Long.parseLong(ss[0]);
interval.stop = interval.start + 1;
break;
case 2:
interval.start = Long.parseLong(ss[0]);
interval.stop = Long.parseLong(ss[1]) + 1;
break;
default:
throw new RuntimeException(String.format("parseInterval failed due to wrong format: %s", str));
}
return interval;
}
/**
* 把{start,stop}连续的合并掉:
* [{start:1, stop:4},{start:4, stop:5}] => [{start:1, stop:5}]
*
* @param intervals
* @return
*/
public static List<Interval> combine(List<Interval> intervals) {
List<Interval> combined = new ArrayList<Interval>();
Collections.sort(intervals);
int len = intervals.size();
for (int i=0; i<len; i++) {
combined.add(intervals.get(i));
int j;
for (j=i+1; j<len; j++) {
if (intervals.get(i).stop >= intervals.get(j).start) {
intervals.get(i).stop = intervals.get(j).stop;
} else {
break;
}
}
i = j - 1;
}
return combined;
}
}

68
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpGTIDCommandPacket.java Normal file
View File

@@ -0,0 +1,68 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* Created by hiwjd on 2018/4/24.
* hiwjd0@gmail.com
*
* https://dev.mysql.com/doc/internals/en/com-binlog-dump-gtid.html
*/
public class BinlogDumpGTIDCommandPacket extends CommandPacket {
public static final int BINLOG_DUMP_NON_BLOCK = 0x01;
public static final int BINLOG_THROUGH_POSITION = 0x02;
public static final int BINLOG_THROUGH_GTID = 0x04;
public long slaveServerId;
public GTIDSet gtidSet;
public BinlogDumpGTIDCommandPacket() {
setCommand((byte) 0x1e);
}
@Override
public void fromBytes(byte[] data) throws IOException {
}
@Override
public byte[] toBytes() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
// 0. [1] write command number
out.write(getCommand());
// 1. [2] flags
ByteHelper.writeUnsignedShortLittleEndian(BINLOG_THROUGH_GTID, out);
// 2. [4] server-id
ByteHelper.writeUnsignedIntLittleEndian(slaveServerId, out);
// 3. [4] binlog-filename-len
ByteHelper.writeUnsignedIntLittleEndian(0, out);
// 4. [] binlog-filename
// skip
// 5. [8] binlog-pos
ByteHelper.writeUnsignedInt64LittleEndian(4, out);
// if flags & BINLOG_THROUGH_GTID {
byte[] bs = gtidSet.encode();
// 6. [4] data-size
ByteHelper.writeUnsignedIntLittleEndian(bs.length, out);
// 7, [] data
// [8] n_sids // 文档写的是4个字节，其实是8个字节
// for n_sids {
// [16] SID
// [8] n_intervals
// for n_intervals {
// [8] start (signed)
// [8] end (signed)
// }
// }
out.write(bs);
// }
return out.toByteArray();
}
}

11
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java
View File

@@ -127,6 +127,17 @@ public abstract class ByteHelper {
out.write((byte) (data >>> 24));
}
public static void writeUnsignedInt64LittleEndian(long data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) (data >>> 8));
out.write((byte) (data >>> 16));
out.write((byte) (data >>> 24));
out.write((byte) (data >>> 32));
out.write((byte) (data >>> 40));
out.write((byte) (data >>> 48));
out.write((byte) (data >>> 56));
}
public static void writeUnsignedShortLittleEndian(int data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) ((data >>> 8) & 0xFF));

129
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java Normal file
View File

@@ -0,0 +1,129 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.UUIDSet;
import org.junit.Test;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.*;
/**
* Created by hiwjd on 2018/4/25.
* hiwjd0@gmail.com
*/
public class MysqlGTIDSetTest {
@Test
public void testEncode() throws IOException {
GTIDSet gtidSet = MysqlGTIDSet.parse("726757ad-4455-11e8-ae04-0242ac110002:1");
byte[] bytes = gtidSet.encode();
byte[] expected = {
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x67, 0x57, (byte)0xad,
0x44, 0x55, 0x11, (byte)0xe8, (byte)0xae, 0x04, 0x02, 0x42, (byte)0xac, 0x11, 0x00, 0x02,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
for (int i=0; i<bytes.length; i++) {
assertEquals(expected[i], bytes[i]);
}
}
@Test
public void testParse() {
Map<String, MysqlGTIDSet> cases = new HashMap<String, MysqlGTIDSet>(5);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 2))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3:4",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 5))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4, 7, 10))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3,726757ad-4455-11e8-ae04-0242ac110003:4",
buildForTest(
Arrays.asList(
new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4),
new Material("726757ad-4455-11e8-ae04-0242ac110003", 4, 5)
)
)
);
for (Map.Entry<String, MysqlGTIDSet> entry : cases.entrySet()) {
MysqlGTIDSet expected = entry.getValue();
MysqlGTIDSet actual = MysqlGTIDSet.parse(entry.getKey());
assertEquals(expected, actual);
}
}
private static class Material {
public Material(String uuid, long start, long stop) {
this.uuid = uuid;
this.start = start;
this.stop = stop;
this.start1 = 0;
this.stop1 = 0;
}
public Material(String uuid, long start, long stop, long start1, long stop1) {
this.uuid = uuid;
this.start = start;
this.stop = stop;
this.start1 = start1;
this.stop1 = stop1;
}
public String uuid;
public long start;
public long stop;
public long start1;
public long stop1;
}
private MysqlGTIDSet buildForTest(Material material) {
return buildForTest(Arrays.asList(material));
}
private MysqlGTIDSet buildForTest(List<Material> materials) {
Map<String, UUIDSet> sets = new HashMap<String, UUIDSet>();
for (Material a : materials) {
UUIDSet.Interval interval = new UUIDSet.Interval();
interval.start = a.start;
interval.stop = a.stop;
List<UUIDSet.Interval> intervals = new ArrayList<UUIDSet.Interval>();
intervals.add(interval);
if (a.start1 > 0 && a.stop1 > 0) {
UUIDSet.Interval interval1 = new UUIDSet.Interval();
interval1.start = a.start1;
interval1.stop = a.stop1;
intervals.add(interval1);
}
UUIDSet us = new UUIDSet();
us.SID = UUID.fromString(a.uuid);
us.intervals = intervals;
sets.put(a.uuid, us);
}
MysqlGTIDSet gs = new MysqlGTIDSet();
gs.sets = sets;
return gs;
}
}

30
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/UUIDSetTest.java Normal file
View File

@@ -0,0 +1,30 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import com.alibaba.otter.canal.parse.driver.mysql.packets.UUIDSet;
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.*;
/**
* Created by hiwjd on 2018/4/26.
* hiwjd0@gmail.com
*/
public class UUIDSetTest {
@Test
public void testToString() {
Map<String, String> cases = new HashMap<String, String>(4);
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1", "726757ad-4455-11e8-ae04-0242ac110002:1");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3", "726757ad-4455-11e8-ae04-0242ac110002:1-3");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:4-6", "726757ad-4455-11e8-ae04-0242ac110002:1-6");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:5-7", "726757ad-4455-11e8-ae04-0242ac110002:1-3:5-7");
for (Map.Entry<String, String> entry : cases.entrySet()) {
String expected = entry.getValue();
assertEquals(expected, UUIDSet.parse(entry.getKey()).toString());
}
}
}