chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

1. Add MariaDB gtid support 2. fix issue:#561 增加一个外部参数控制，允许位点不存在时自动重置到当前位点 (#3025)

Browse Source 
* fix issue:#561 增加一个外部参数控制，允许位点不存在时自动重置到当前位点

* Add MariaDB GTID support
This commit is contained in:
文哥
2020-09-24 17:56:40 +08:00
committed by GitHub
parent 09429c241f
commit de1dd70609
18 changed files with 337 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java
View File

@@ -24,7 +24,7 @@ public final class LogContext {
private GTIDSet gtidSet;
private GtidLogEvent gtidLogEvent; // save current gtid log event
private LogEvent gtidLogEvent; // save current gtid log event
public LogContext(){
this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
@@ -75,11 +75,11 @@ public final class LogContext {
this.gtidSet = gtidSet;
}
public GtidLogEvent getGtidLogEvent() {
public LogEvent getGtidLogEvent() {
return gtidLogEvent;
}
public void setGtidLogEvent(GtidLogEvent gtidLogEvent) {
public void setGtidLogEvent(LogEvent gtidLogEvent) {
this.gtidLogEvent = gtidLogEvent;
}
}

49
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
View File

@@ -162,7 +162,7 @@ public final class LogDecoder {
buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
}
GTIDSet gtidSet = context.getGtidSet();
GtidLogEvent gtidLogEvent = context.getGtidLogEvent();
LogEvent gtidLogEvent = context.getGtidLogEvent();
switch (header.getType()) {
case LogEvent.QUERY_EVENT: {
QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
@@ -185,7 +185,8 @@ public final class LogDecoder {
context.putTable(mapEvent);
return mapEvent;
}
case LogEvent.WRITE_ROWS_EVENT_V1: {
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT: {
RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
@@ -193,7 +194,8 @@ public final class LogDecoder {
header.putGtid(context.getGtidSet(), gtidLogEvent);
return event;
}
case LogEvent.UPDATE_ROWS_EVENT_V1: {
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.UPDATE_ROWS_EVENT: {
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
@@ -201,7 +203,8 @@ public final class LogDecoder {
header.putGtid(context.getGtidSet(), gtidLogEvent);
return event;
}
case LogEvent.DELETE_ROWS_EVENT_V1: {
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT: {
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
@@ -349,30 +352,6 @@ public final class LogDecoder {
header.putGtid(context.getGtidSet(), gtidLogEvent);
return event;
}
case LogEvent.WRITE_ROWS_EVENT: {
RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
header.putGtid(context.getGtidSet(), gtidLogEvent);
return event;
}
case LogEvent.UPDATE_ROWS_EVENT: {
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
header.putGtid(context.getGtidSet(), gtidLogEvent);
return event;
}
case LogEvent.DELETE_ROWS_EVENT: {
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
header.putGtid(context.getGtidSet(), gtidLogEvent);
return event;
}
case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: {
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent, true);
/* updating position in context */
@@ -436,12 +415,26 @@ public final class LogDecoder {
MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
if (gtidSet != null) {
gtidSet.update(event.getGtidStr());
// update latest gtid
header.putGtid(gtidSet, event);
}
// update current gtid event to context
context.setGtidLogEvent(event);
return event;
}
case LogEvent.GTID_LIST_EVENT: {
MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
if (gtidSet != null) {
gtidSet.update(event.getGtidStr());
// update latest gtid
header.putGtid(gtidSet, event);
}
// update current gtid event to context
context.setGtidLogEvent(event);
return event;
}
case LogEvent.START_ENCRYPTION_EVENT: {

5
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java
View File

@@ -316,10 +316,11 @@ public final class LogHeader {
return gtidMap.get(CURRENT_GTID_LAST_COMMIT);
}
public void putGtid(GTIDSet gtidSet, GtidLogEvent event) {
public void putGtid(GTIDSet gtidSet, LogEvent gtidEvent) {
if (gtidSet != null) {
gtidMap.put(GTID_SET_STRING, gtidSet.toString());
if (event != null) {
if (gtidEvent != null && gtidEvent instanceof GtidLogEvent) {
GtidLogEvent event = (GtidLogEvent)gtidEvent;
gtidMap.put(CURRENT_GTID_STRING, event.getGtidStr());
gtidMap.put(CURRENT_GTID_SN, String.valueOf(event.getSequenceNumber()));
gtidMap.put(CURRENT_GTID_LAST_COMMIT, String.valueOf(event.getLastCommitted()));

34
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java
View File

@@ -1,6 +1,9 @@
package com.taobao.tddl.dbsync.binlog.event.mariadb;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MariaGTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MariaGtid;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
@@ -11,11 +14,36 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
* @author jianghang 2014-1-20 下午4:51:50
* @since 1.0.17
*/
public class MariaGtidListLogEvent extends IgnorableLogEvent {
public class MariaGtidListLogEvent extends LogEvent {
private MariaGTIDSet mariaGTIDSet;
/**
* <pre>
* mariadb gtidListLog event format
* uint<4> Number of GTIDs
* GTID[0]
* uint<4> Replication Domain ID
* uint<4> Server_ID
* uint<8> GTID sequence ...
* GTID[n]
* </pre>
* @param header
* @param buffer
* @param descriptionEvent
*/
public MariaGtidListLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
// do nothing , just ignore log event
super(header);
long gtidLenth = buffer.getUint32();
mariaGTIDSet = new MariaGTIDSet();
for (int i = 0; i < gtidLenth; i++) {
long domainId = buffer.getUint32();
long serverId = buffer.getUint32();
long sequence = buffer.getUlong64().longValue();
mariaGTIDSet.add(new MariaGtid(domainId, serverId, sequence));
}
}
public String getGtidStr() {
return mariaGTIDSet.toString();
}
}

19
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java
View File

@@ -1,6 +1,8 @@
package com.taobao.tddl.dbsync.binlog.event.mariadb;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MariaGtid;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
@@ -11,9 +13,9 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
* @author jianghang 2014-1-20 下午4:49:10
* @since 1.0.17
*/
public class MariaGtidLogEvent extends IgnorableLogEvent {
public class MariaGtidLogEvent extends LogEvent {
private long gtid;
private MariaGtid mariaGtid;
/**
* <pre>
@@ -30,13 +32,14 @@ public class MariaGtidLogEvent extends IgnorableLogEvent {
*/
public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
gtid = buffer.getUlong64().longValue();
// do nothing , just ignore log event
super(header);
long sequence = buffer.getUlong64().longValue();
long domainId = buffer.getUint32();
long serverId = header.getServerId();
mariaGtid = new MariaGtid(domainId, serverId, sequence);
}
public long getGtid() {
return gtid;
public String getGtidStr() {
return mariaGtid.toString();
}
}