fixed format

This commit is contained in:
七锋
2018-06-14 19:25:34 +08:00
parent dca1b20043
commit a4255a6377
38 changed files with 11024 additions and 11392 deletions

View File

@@ -53,7 +53,7 @@ public class MysqlConnector {
String addr = address.getHostString();
int port = address.getPort();
this.address = new InetSocketAddress(addr, port);
this.username = username;
this.password = password;
}

View File

@@ -3,8 +3,7 @@ package com.alibaba.otter.canal.parse.driver.mysql.packets;
import java.io.IOException;
/**
* Created by hiwjd on 2018/4/23.
* hiwjd0@gmail.com
* Created by hiwjd on 2018/4/23. hiwjd0@gmail.com
*/
public interface GTIDSet {
@@ -17,6 +16,7 @@ public interface GTIDSet {
/**
* 更新当前实例
*
* @param str
* @throws Exception
*/

View File

@@ -1,14 +1,14 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.HashMap;
import java.util.Map;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
/**
* Created by hiwjd on 2018/4/23.
* hiwjd0@gmail.com
* Created by hiwjd on 2018/4/23. hiwjd0@gmail.com
*/
public class MysqlGTIDSet implements GTIDSet {
@@ -56,61 +56,26 @@ public class MysqlGTIDSet implements GTIDSet {
}
/**
* 解析如下格式的字符串为MysqlGTIDSet:
*
* 726757ad-4455-11e8-ae04-0242ac110002:1 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:2}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:4}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:4 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:5}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:4}, {start:7, stop: 10}]
* }
* }
* }
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3,726757ad-4455-11e8-ae04-0242ac110003:4 =>
* MysqlGTIDSet{
* sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:1, stop:4}]
* },
* 726757ad-4455-11e8-ae04-0242ac110003: UUIDSet{
* SID: 726757ad-4455-11e8-ae04-0242ac110002,
* intervals: [{start:4, stop:5}]
* }
* }
* }
* 解析如下格式的字符串为MysqlGTIDSet: 726757ad-4455-11e8-ae04-0242ac110002:1 =>
* MysqlGTIDSet{ sets: { 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{ SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:2}] } }
* } 726757ad-4455-11e8-ae04-0242ac110002:1-3 => MysqlGTIDSet{ sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{ SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}] } }
* } 726757ad-4455-11e8-ae04-0242ac110002:1-3:4 => MysqlGTIDSet{ sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{ SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:5}] } }
* } 726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9 => MysqlGTIDSet{ sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{ SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4},
* {start:7, stop: 10}] } } }
* 726757ad-4455-11e8-ae04-0242ac110002:1-3,726757
* ad-4455-11e8-ae04-0242ac110003:4 => MysqlGTIDSet{ sets: {
* 726757ad-4455-11e8-ae04-0242ac110002: UUIDSet{ SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}] },
* 726757ad-4455-11e8-ae04-0242ac110003: UUIDSet{ SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:4, stop:5}] } }
* }
*
* @param gtidData
* @return

View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -10,13 +8,14 @@ import java.util.Collections;
import java.util.List;
import java.util.UUID;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
/**
* Created by hiwjd on 2018/4/23.
* hiwjd0@gmail.com
* Created by hiwjd on 2018/4/23. hiwjd0@gmail.com
*/
public class UUIDSet {
public UUID SID;
public UUID SID;
public List<Interval> intervals;
public byte[] encode() throws IOException {
@@ -54,6 +53,7 @@ public class UUIDSet {
}
public static class Interval implements Comparable<Interval> {
public long start;
public long stop;
@@ -85,19 +85,15 @@ public class UUIDSet {
}
/**
* 解析如下格式字符串为UUIDSet:
*
* 726757ad-4455-11e8-ae04-0242ac110002:1 =>
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:2}]}
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3 =>
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}]}
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:4
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:5}]}
*
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9
* UUIDSet{SID: 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}, {start:7, stop:10}]}
* 解析如下格式字符串为UUIDSet: 726757ad-4455-11e8-ae04-0242ac110002:1 => UUIDSet{SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:2}]}
* 726757ad-4455-11e8-ae04-0242ac110002:1-3 => UUIDSet{SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4}]}
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:4 UUIDSet{SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:5}]}
* 726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9 UUIDSet{SID:
* 726757ad-4455-11e8-ae04-0242ac110002, intervals: [{start:1, stop:4},
* {start:7, stop:10}]}
*
* @param str
* @return
@@ -110,7 +106,7 @@ public class UUIDSet {
}
List<Interval> intervals = new ArrayList<Interval>();
for (int i=1; i<ss.length; i++) {
for (int i = 1; i < ss.length; i++) {
intervals.add(parseInterval(ss[i]));
}
@@ -134,7 +130,7 @@ public class UUIDSet {
sb.append(":");
sb.append(interval.start);
sb.append("-");
sb.append(interval.stop-1);
sb.append(interval.stop - 1);
}
}
@@ -142,12 +138,8 @@ public class UUIDSet {
}
/**
* 解析如下格式字符串为Interval:
*
* 1 => Interval{start:1, stop:2}
* 1-3 => Interval{start:1, stop:4}
*
* 注意!字符串格式表达时[n,m]是两侧都包含的,Interval表达时[n,m)右侧开
* 解析如下格式字符串为Interval: 1 => Interval{start:1, stop:2} 1-3 =>
* Interval{start:1, stop:4} 注意!字符串格式表达时[n,m]是两侧都包含的,Interval表达时[n,m)右侧开
*
* @param str
* @return
@@ -173,8 +165,8 @@ public class UUIDSet {
}
/**
* 把{start,stop}连续的合并掉:
* [{start:1, stop:4},{start:4, stop:5}] => [{start:1, stop:5}]
* 把{start,stop}连续的合并掉: [{start:1, stop:4},{start:4, stop:5}] => [{start:1,
* stop:5}]
*
* @param intervals
* @return
@@ -183,11 +175,11 @@ public class UUIDSet {
List<Interval> combined = new ArrayList<Interval>();
Collections.sort(intervals);
int len = intervals.size();
for (int i=0; i<len; i++) {
for (int i = 0; i < len; i++) {
combined.add(intervals.get(i));
int j;
for (j=i+1; j<len; j++) {
for (j = i + 1; j < len; j++) {
if (intervals.get(i).stop >= intervals.get(j).start) {
intervals.get(i).stop = intervals.get(j).stop;
} else {

View File

@@ -1,28 +1,26 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* Created by hiwjd on 2018/4/24.
* hiwjd0@gmail.com
*
* Created by hiwjd on 2018/4/24. hiwjd0@gmail.com
* https://dev.mysql.com/doc/internals/en/com-binlog-dump-gtid.html
*/
public class BinlogDumpGTIDCommandPacket extends CommandPacket {
public static final int BINLOG_DUMP_NON_BLOCK = 0x01;
public static final int BINLOG_DUMP_NON_BLOCK = 0x01;
public static final int BINLOG_THROUGH_POSITION = 0x02;
public static final int BINLOG_THROUGH_GTID = 0x04;
public static final int BINLOG_THROUGH_GTID = 0x04;
public long slaveServerId;
public GTIDSet gtidSet;
public long slaveServerId;
public GTIDSet gtidSet;
public BinlogDumpGTIDCommandPacket() {
public BinlogDumpGTIDCommandPacket(){
setCommand((byte) 0x1e);
}
@@ -51,15 +49,15 @@ public class BinlogDumpGTIDCommandPacket extends CommandPacket {
// 6. [4] data-size
ByteHelper.writeUnsignedIntLittleEndian(bs.length, out);
// 7, [] data
// [8] n_sids // 文档写的是4个字节,其实是8个字节
// for n_sids {
// [16] SID
// [8] n_intervals
// for n_intervals {
// [8] start (signed)
// [8] end (signed)
// }
// }
// [8] n_sids // 文档写的是4个字节,其实是8个字节
// for n_sids {
// [16] SID
// [8] n_intervals
// for n_intervals {
// [8] start (signed)
// [8] end (signed)
// }
// }
out.write(bs);
// }

View File

@@ -29,8 +29,8 @@ import org.slf4j.LoggerFactory;
@SuppressWarnings({ "rawtypes", "deprecation" })
public abstract class NettySocketChannelPool {
private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组
private static Bootstrap boot = new Bootstrap(); // 主
private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组
private static Bootstrap boot = new Bootstrap(); // 主
private static Map<Channel, SocketChannel> chManager = new ConcurrentHashMap<Channel, SocketChannel>();
private static final Logger logger = LoggerFactory.getLogger(NettySocketChannelPool.class);

View File

@@ -108,7 +108,7 @@ public abstract class ByteHelper {
return out.toByteArray();
}
public static void write8ByteUnsignedIntLittleEndian(long data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) (data >>> 8));
@@ -119,7 +119,7 @@ public abstract class ByteHelper {
out.write((byte) (data >>> 48));
out.write((byte) (data >>> 56));
}
public static void writeUnsignedIntLittleEndian(long data, ByteArrayOutputStream out) {
out.write((byte) (data & 0xFF));
out.write((byte) (data >>> 8));

View File

@@ -14,15 +14,15 @@ public abstract class PacketManager {
}
public static HeaderPacket readHeader(SocketChannel ch, int len, int timeout) throws IOException {
HeaderPacket header = new HeaderPacket();
header.fromBytes(ch.read(len, timeout));
return header;
HeaderPacket header = new HeaderPacket();
header.fromBytes(ch.read(len, timeout));
return header;
}
public static byte[] readBytes(SocketChannel ch, int len) throws IOException {
return ch.read(len);
}
public static byte[] readBytes(SocketChannel ch, int len, int timeout) throws IOException {
return ch.read(len, timeout);
}

View File

@@ -1,17 +1,23 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.junit.Test;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.UUIDSet;
import org.junit.Test;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.*;
/**
* Created by hiwjd on 2018/4/25.
* hiwjd0@gmail.com
* Created by hiwjd on 2018/4/25. hiwjd0@gmail.com
*/
public class MysqlGTIDSetTest {
@@ -20,14 +26,12 @@ public class MysqlGTIDSetTest {
GTIDSet gtidSet = MysqlGTIDSet.parse("726757ad-4455-11e8-ae04-0242ac110002:1");
byte[] bytes = gtidSet.encode();
byte[] expected = {
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x67, 0x57, (byte)0xad,
0x44, 0x55, 0x11, (byte)0xe8, (byte)0xae, 0x04, 0x02, 0x42, (byte)0xac, 0x11, 0x00, 0x02,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
byte[] expected = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x67, 0x57, (byte) 0xad, 0x44, 0x55,
0x11, (byte) 0xe8, (byte) 0xae, 0x04, 0x02, 0x42, (byte) 0xac, 0x11, 0x00, 0x02, 0x01, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00 };
for (int i=0; i<bytes.length; i++) {
for (int i = 0; i < bytes.length; i++) {
assertEquals(expected[i], bytes[i]);
}
}
@@ -35,31 +39,17 @@ public class MysqlGTIDSetTest {
@Test
public void testParse() {
Map<String, MysqlGTIDSet> cases = new HashMap<String, MysqlGTIDSet>(5);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 2))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3:4",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 5))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4, 7, 10))
);
cases.put(
"726757ad-4455-11e8-ae04-0242ac110002:1-3,726757ad-4455-11e8-ae04-0242ac110003:4",
buildForTest(
Arrays.asList(
new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4),
new Material("726757ad-4455-11e8-ae04-0242ac110003", 4, 5)
)
)
);
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 2)));
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4)));
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:4",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 5)));
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:7-9",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4, 7, 10)));
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3,726757ad-4455-11e8-ae04-0242ac110003:4",
buildForTest(Arrays.asList(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 4),
new Material("726757ad-4455-11e8-ae04-0242ac110003", 4, 5))));
for (Map.Entry<String, MysqlGTIDSet> entry : cases.entrySet()) {
MysqlGTIDSet expected = entry.getValue();
@@ -71,7 +61,7 @@ public class MysqlGTIDSetTest {
private static class Material {
public Material(String uuid, long start, long stop) {
public Material(String uuid, long start, long stop){
this.uuid = uuid;
this.start = start;
this.stop = stop;
@@ -79,7 +69,7 @@ public class MysqlGTIDSetTest {
this.stop1 = 0;
}
public Material(String uuid, long start, long stop, long start1, long stop1) {
public Material(String uuid, long start, long stop, long start1, long stop1){
this.uuid = uuid;
this.start = start;
this.stop = stop;
@@ -88,10 +78,10 @@ public class MysqlGTIDSetTest {
}
public String uuid;
public long start;
public long stop;
public long start1;
public long stop1;
public long start;
public long stop;
public long start1;
public long stop1;
}
private MysqlGTIDSet buildForTest(Material material) {

View File

@@ -1,16 +1,16 @@
package com.alibaba.otter.canal.parse.driver.mysql;
import com.alibaba.otter.canal.parse.driver.mysql.packets.UUIDSet;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.*;
import org.junit.Test;
import com.alibaba.otter.canal.parse.driver.mysql.packets.UUIDSet;
/**
* Created by hiwjd on 2018/4/26.
* hiwjd0@gmail.com
* Created by hiwjd on 2018/4/26. hiwjd0@gmail.com
*/
public class UUIDSetTest {