chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixes issue #4388 , support mariadb 10.x log_bin_compress

Browse Source
This commit is contained in:
jianghang.loujh
2023-09-01 13:42:49 +08:00
parent 252ca0279f
commit f13cc4ee9f
23 changed files with 673 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
View File

@@ -114,17 +114,26 @@ public class LocalBinLogConnection implements ErosaConnection {
continue;
}
checkServerId(event);
if (!func.sink(event)) {
needContinue = false;
break;
List<LogEvent> iterateEvents = decoder.processIterateDecode(event, context);
if (!iterateEvents.isEmpty()) {
// 处理compress event
for(LogEvent itEvent : iterateEvents) {
if (!func.sink(event)) {
needContinue = false;
break;
}
}
} else {
if (!func.sink(event)) {
needContinue = false;
break;
}
}
}
fetcher.close(); // 关闭上一个文件
parserFinish(current.getName());
if (needContinue) {// 读取下一个
File nextFile;
if (needWait) {
nextFile = binlogs.waitForNextFile(current);

14
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -213,8 +213,18 @@ public class MysqlConnection implements ErosaConnection {
throw new CanalParseException("parse failed");
}
if (!func.sink(event)) {
break;
List<LogEvent> iterateEvents = decoder.processIterateDecode(event, context);
if (!iterateEvents.isEmpty()) {
// 处理compress event
for(LogEvent itEvent : iterateEvents) {
if (!func.sink(event)) {
break;
}
}
} else {
if (!func.sink(event)) {
break;
}
}
}
}

214
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
View File

@@ -1,11 +1,14 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
import org.apache.commons.compress.utils.Lists;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
@@ -16,27 +19,12 @@ import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.lmax.disruptor.BatchEventProcessor;
import com.lmax.disruptor.BlockingWaitStrategy;
import com.lmax.disruptor.EventFactory;
import com.lmax.disruptor.EventHandler;
import com.lmax.disruptor.ExceptionHandler;
import com.lmax.disruptor.InsufficientCapacityException;
import com.lmax.disruptor.LifecycleAware;
import com.lmax.disruptor.RingBuffer;
import com.lmax.disruptor.Sequence;
import com.lmax.disruptor.SequenceBarrier;
import com.lmax.disruptor.WorkHandler;
import com.lmax.disruptor.WorkerPool;
import com.lmax.disruptor.*;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.*;
/**
* 针对解析器提供一个多阶段协同的处理
@@ -271,48 +259,71 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
}
int eventType = logEvent.getHeader().getType();
TableMeta tableMeta = null;
boolean needDmlParse = false;
switch (eventType) {
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
if (!filterDmlInsert) {
tableMeta = logEventConvert.parseRowsEventForTableMeta((WriteRowsLogEvent) logEvent);
needDmlParse = true;
}
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
case LogEvent.UPDATE_ROWS_EVENT:
if (!filterDmlUpdate) {
tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
needDmlParse = true;
}
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
if (!filterDmlDelete) {
tableMeta = logEventConvert.parseRowsEventForTableMeta((DeleteRowsLogEvent) logEvent);
needDmlParse = true;
}
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
needDmlParse = true;
break;
default:
CanalEntry.Entry entry = logEventConvert.parse(event.getEvent(), false);
event.setEntry(entry);
}
boolean needIterate = false;
// 记录一下DML的表结构
event.setNeedDmlParse(needDmlParse);
event.setTable(tableMeta);
if (eventType == LogEvent.TRANSACTION_PAYLOAD_EVENT) {
// https://github.com/alibaba/canal/issues/4388
List<LogEvent> deLogEvents = decoder.processIterateDecode(logEvent, context);
List<TableMeta> tableMetas = Lists.newArrayList();
event.setNeedIterate(true);
for (LogEvent deLogEvent : deLogEvents) {
TableMeta table = processEvent(deLogEvent, event);
tableMetas.add(table);
}
event.setIterateEvents(deLogEvents);
event.setIterateTables(tableMetas);
} else {
TableMeta table = processEvent(logEvent, event);
event.setTable(table);
}
} catch (Throwable e) {
exception = new CanalParseException(e);
throw exception;
}
}
private TableMeta processEvent(LogEvent logEvent, MessageEvent event) {
TableMeta tableMeta = null;
boolean needDmlParse = false;
int eventType = logEvent.getHeader().getType();
switch (eventType) {
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
if (!filterDmlInsert) {
tableMeta = logEventConvert.parseRowsEventForTableMeta((WriteRowsLogEvent) logEvent);
needDmlParse = true;
}
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
case LogEvent.UPDATE_ROWS_EVENT:
if (!filterDmlUpdate) {
tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
needDmlParse = true;
}
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
if (!filterDmlDelete) {
tableMeta = logEventConvert.parseRowsEventForTableMeta((DeleteRowsLogEvent) logEvent);
needDmlParse = true;
}
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
needDmlParse = true;
break;
default:
CanalEntry.Entry entry = logEventConvert.parse(event.getEvent(), false);
event.setEntry(entry);
}
// 记录一下DML的表结构
if (needDmlParse && !event.isNeedDmlParse()) {
event.setNeedDmlParse(true);
}
return tableMeta;
}
@Override
public void onStart() {
@@ -330,18 +341,21 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
public void onEvent(MessageEvent event) throws Exception {
try {
if (event.isNeedDmlParse()) {
int eventType = event.getEvent().getHeader().getType();
CanalEntry.Entry entry = null;
switch (eventType) {
case LogEvent.ROWS_QUERY_LOG_EVENT:
entry = logEventConvert.parse(event.getEvent(), false);
break;
default:
// 单独解析dml事件
entry = logEventConvert.parseRowsEvent((RowsLogEvent) event.getEvent(), event.getTable());
if (event.isNeedIterate()) {
// compress binlog
List<CanalEntry.Entry> entrys = Lists.newArrayList();
for (int index = 0; index < event.getIterateEvents().size(); index++) {
CanalEntry.Entry entry = processEvent(event.getIterateEvents().get(index),
event.getIterateTables().get(index));
if (entry != null) {
entrys.add(entry);
}
}
event.setIterateEntrys(entrys);
} else {
CanalEntry.Entry entry = processEvent(event.getEvent(), event.getTable());
event.setEntry(entry);
}
event.setEntry(entry);
}
} catch (Throwable e) {
exception = new CanalParseException(e);
@@ -349,6 +363,29 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
}
}
private CanalEntry.Entry processEvent(LogEvent logEvent, TableMeta table) {
int eventType = logEvent.getHeader().getType();
CanalEntry.Entry entry = null;
switch (eventType) {
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
case LogEvent.UPDATE_ROWS_EVENT:
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
// 单独解析dml事件
entry = logEventConvert.parseRowsEvent((RowsLogEvent) logEvent, table);
break;
default:
// 如果出现compress binlog,会出现其他的event type类型
entry = logEventConvert.parse(logEvent, false);
break;
}
return entry;
}
@Override
public void onStart() {
@@ -364,8 +401,15 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
try {
if (event.getEntry() != null) {
transactionBuffer.add(event.getEntry());
if (event.isNeedIterate()) {
// compress binlog
for (CanalEntry.Entry entry : event.getIterateEntrys()) {
transactionBuffer.add(entry);
}
} else {
if (event.getEntry() != null) {
transactionBuffer.add(event.getEntry());
}
}
LogEvent logEvent = event.getEvent();
@@ -380,7 +424,11 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
event.setEvent(null);
event.setTable(null);
event.setEntry(null);
// clear compress binlog events
event.setNeedDmlParse(false);
event.setIterateEntrys(null);
event.setIterateTables(null);
event.setIterateEvents(null);
} catch (Throwable e) {
exception = new CanalParseException(e);
throw exception;
@@ -405,6 +453,11 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
private boolean needDmlParse = false;
private TableMeta table;
private LogEvent event;
private boolean needIterate = false;
// compress binlog
private List<LogEvent> iterateEvents;
private List<TableMeta> iterateTables;
private List<CanalEntry.Entry> iterateEntrys;
public LogBuffer getBuffer() {
return buffer;
@@ -446,6 +499,37 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
this.table = table;
}
public boolean isNeedIterate() {
return needIterate;
}
public void setNeedIterate(boolean needIterate) {
this.needIterate = needIterate;
}
public List<LogEvent> getIterateEvents() {
return iterateEvents;
}
public List<TableMeta> getIterateTables() {
return iterateTables;
}
public void setIterateEvents(List<LogEvent> iterateEvents) {
this.iterateEvents = iterateEvents;
}
public void setIterateTables(List<TableMeta> iterateTables) {
this.iterateTables = iterateTables;
}
public List<CanalEntry.Entry> getIterateEntrys() {
return iterateEntrys;
}
public void setIterateEntrys(List<CanalEntry.Entry> iterateEntrys) {
this.iterateEntrys = iterateEntrys;
}
}
static class SimpleFatalExceptionHandler implements ExceptionHandler {