chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

1. Add MariaDB gtid support 2. fix issue:#561 增加一个外部参数控制，允许位点不存在时自动重置到当前位点 (#3025)

Browse Source 
* fix issue:#561 增加一个外部参数控制，允许位点不存在时自动重置到当前位点

* Add MariaDB GTID support
This commit is contained in:
文哥
2020-09-24 17:56:40 +08:00
committed by GitHub
parent 09429c241f
commit de1dd70609
18 changed files with 337 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
View File

@@ -10,6 +10,7 @@ import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.math.RandomUtils;
@@ -22,7 +23,6 @@ import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
@@ -39,6 +39,8 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
import static com.alibaba.otter.canal.parse.driver.mysql.utils.GtidUtil.parseGtidSet;
/**
* 抽象的EventParser, 最大化共用mysql/oracle版本的实现
*
@@ -160,6 +162,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
public void run() {
MDC.put("destination", String.valueOf(destination));
ErosaConnection erosaConnection = null;
boolean isMariaDB = false;
while (running) {
try {
// 开始执行replication
@@ -178,6 +181,10 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
if (queryServerId != 0) {
serverId = queryServerId;
}
if (erosaConnection instanceof MysqlConnection) {
isMariaDB = ((MysqlConnection)erosaConnection).isMariaDB();
}
// 4. 获取最后的位置信息
long start = System.currentTimeMillis();
logger.warn("---> begin to find start position, it will be long time for reset or first position");
@@ -242,7 +249,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
multiStageCoprocessor = buildMultiStageCoprocessor();
if (isGTIDMode() && StringUtils.isNotEmpty(startPosition.getGtid())) {
// 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
GTIDSet gtidSet = MysqlGTIDSet.parse(startPosition.getGtid());
GTIDSet gtidSet = parseGtidSet(startPosition.getGtid(),isMariaDB);
((MysqlMultiStageCoprocessor) multiStageCoprocessor).setGtidSet(gtidSet);
multiStageCoprocessor.start();
erosaConnection.dump(gtidSet, multiStageCoprocessor);
@@ -260,7 +267,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
} else {
if (isGTIDMode() && StringUtils.isNotEmpty(startPosition.getGtid())) {
// 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
erosaConnection.dump(MysqlGTIDSet.parse(startPosition.getGtid()), sinkHandler);
erosaConnection.dump(parseGtidSet(startPosition.getGtid(), isMariaDB), sinkHandler);
} else {
if (StringUtils.isEmpty(startPosition.getJournalName())
&& startPosition.getTimestamp() != null) {
@@ -715,6 +722,4 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
public Map<String, List<String>> getFieldBlackFilterMap() {
return fieldBlackFilterMap;
}
}

32
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import static com.alibaba.otter.canal.parse.driver.mysql.utils.GtidUtil.parseGtidSet;
import static com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher.MASTER_HEARTBEAT_PERIOD_SECONDS;
import java.io.IOException;
@@ -137,8 +138,14 @@ public class MysqlConnection implements ErosaConnection {
// 'errno = 1236, sqlstate = HY000 errmsg = The slave is connecting
// using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
if (StringUtils.isNotEmpty(gtid)) {
decoder.handle(LogEvent.GTID_LOG_EVENT);
context.setGtidSet(MysqlGTIDSet.parse(gtid));
GTIDSet gtidSet = parseGtidSet(gtid, isMariaDB());
if (isMariaDB()) {
decoder.handle(LogEvent.GTID_EVENT);
decoder.handle(LogEvent.GTID_LIST_EVENT);
} else {
decoder.handle(LogEvent.GTID_LOG_EVENT);
}
context.setGtidSet(gtidSet);
}
context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
while (fetcher.fetch()) {
@@ -332,6 +339,14 @@ public class MysqlConnection implements ErosaConnection {
}
private void sendBinlogDumpGTID(GTIDSet gtidSet) throws IOException {
if (isMariaDB()) {
sendMariaBinlogDumpGTID(gtidSet);
return;
}
sendMySQLBinlogDumpGTID(gtidSet);
}
private void sendMySQLBinlogDumpGTID(GTIDSet gtidSet) throws IOException {
BinlogDumpGTIDCommandPacket binlogDumpCmd = new BinlogDumpGTIDCommandPacket();
binlogDumpCmd.slaveServerId = this.slaveId;
binlogDumpCmd.gtidSet = gtidSet;
@@ -345,6 +360,15 @@ public class MysqlConnection implements ErosaConnection {
connector.setDumping(true);
}
private void sendMariaBinlogDumpGTID(GTIDSet gtidSet) throws IOException {
update("SET @slave_connect_state = '" + new String(gtidSet.encode()) + "'");
update("SET @slave_gtid_strict_mode = 0");
update("SET @slave_gtid_ignore_duplicates = 0");
sendRegisterSlave();
sendBinlogDump("", 0L);
connector.setDumping(true);
}
public MysqlConnection fork() {
MysqlConnection connection = new MysqlConnection();
connection.setCharset(getCharset());
@@ -673,4 +697,8 @@ public class MysqlConnection implements ErosaConnection {
this.receivedBinlogBytes = receivedBinlogBytes;
}
public boolean isMariaDB() {
return connector.getServerVersion() != null && connector.getServerVersion().toLowerCase().contains("mariadb");
}
}

24
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -69,6 +69,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private int dumpErrorCount = 0; // binlogDump失败异常计数
private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值
private boolean rdsOssMode = false;
private boolean autoResetLatestPosMode = false; // true: binlog被删除之后，自动按最新的数据订阅
protected ErosaConnection buildErosaConnection() {
return buildMysqlConnection(this.runningInfo);
@@ -484,7 +485,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
dumpErrorCount = 0;
return findPosition;
}
// 处理 binlog 位点被删除的情况，提供自动重置到当前位点的功能
// 应用场景: 测试环境不稳定，位点经常被删。强烈不建议在正式环境中开启此控制参数，因为binlog 丢失调到最新位点也即意味着数据丢失
if (isAutoResetLatestPosMode()) {
dumpErrorCount = 0;
return findEndPosition(mysqlConnection);
}
Long timestamp = logPosition.getPostion().getTimestamp();
if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
// 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
@@ -663,6 +669,14 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
if (isGTIDMode() && fields.size() > 4) {
endPosition.setGtid(fields.get(4));
}
// MariaDB 无法通过`show master status`获取 gtid
if (mysqlConnection.isMariaDB() && isGTIDMode()) {
ResultSetPacket gtidPacket = mysqlConnection.query("SELECT @@global.gtid_binlog_pos");
List<String> gtidFields = gtidPacket.getFieldValues();
if (!CollectionUtils.isEmpty(gtidFields) && gtidFields.size() > 0) {
endPosition.setGtid(gtidFields.get(0));
}
}
return endPosition;
} catch (IOException e) {
throw new CanalParseException("command : 'show master status' has an error!", e);
@@ -938,4 +952,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
public void setDumpErrorCount(int dumpErrorCount) {
this.dumpErrorCount = dumpErrorCount;
}
public boolean isAutoResetLatestPosMode() {
return autoResetLatestPosMode;
}
public void setAutoResetLatestPosMode(boolean autoResetLatestPosMode) {
this.autoResetLatestPosMode = autoResetLatestPosMode;
}
}

34
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -12,6 +12,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger;
@@ -145,7 +147,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return parseGTIDLogEvent((GtidLogEvent) logEvent);
case LogEvent.HEARTBEAT_LOG_EVENT:
return parseHeartbeatLogEvent((HeartbeatLogEvent) logEvent);
case LogEvent.GTID_EVENT:
case LogEvent.GTID_LIST_EVENT:
return parseMariaGTIDLogEvent(logEvent);
default:
break;
}
@@ -186,6 +190,19 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
private Entry parseMariaGTIDLogEvent(LogEvent logEvent) {
LogHeader logHeader = logEvent.getHeader();
Pair.Builder builder = Pair.newBuilder();
builder.setKey("gtid");
if (logEvent instanceof MariaGtidLogEvent) {
builder.setValue(((MariaGtidLogEvent)logEvent).getGtidStr());
} else if (logEvent instanceof MariaGtidListLogEvent) {
builder.setValue(((MariaGtidListLogEvent)logEvent).getGtidStr());
}
Header header = createHeader(logHeader, "", "", EventType.GTID);
return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
private Entry parseQueryEvent(QueryLogEvent event, boolean isSeek) {
String queryString = event.getQuery();
if (StringUtils.startsWithIgnoreCase(queryString, XA_START)) {
@@ -276,16 +293,15 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
Header header = createHeader(event.getHeader(), schemaName, tableName, type);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
if (type == EventType.QUERY && !isDml) {
rowChangeBuider.setIsDdl(true);
}
rowChangeBuider.setSql(queryString);
RowChange.Builder rowChangeBuilder = RowChange.newBuilder();
rowChangeBuilder.setIsDdl(!isDml);
rowChangeBuilder.setSql(queryString);
if (StringUtils.isNotEmpty(event.getDbName())) {// 可能为空
rowChangeBuider.setDdlSchemaName(event.getDbName());
rowChangeBuilder.setDdlSchemaName(event.getDbName());
}
rowChangeBuider.setEventType(type);
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
rowChangeBuilder.setEventType(type);
return createEntry(header, EntryType.ROWDATA, rowChangeBuilder.build().toByteString());
}
}