chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

基于明确的I/U/D是否过滤 (#3452)

Browse Source 
* dmls filter

* dmls filters

* 基于明确的I/U/D是否过滤

* 基于明确的I/U/D是否过滤
This commit is contained in:
suman-lee
2021-04-19 10:10:54 +08:00
committed by GitHub
parent 94425223bd
commit 21904ef1dc
7 changed files with 59 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
View File

@@ -16,6 +16,7 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DefaultTableMetaTSDBFact
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBFactory;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import org.apache.commons.lang.StringUtils;
public abstract class AbstractMysqlEventParser extends AbstractEventParser {
@@ -37,6 +38,10 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
protected boolean filterRows = false;
protected boolean filterTableError = false;
protected boolean useDruidDdlFilter = true;
protected boolean filterDmlInsert = false;
protected boolean filterDmlUpdate = false;
protected boolean filterDmlDelete = false;
// instance received binlog bytes
protected final AtomicLong receivedBinlogBytes = new AtomicLong(0L);
private final AtomicLong eventsPublishBlockingTime = new AtomicLong(0L);
@@ -173,7 +178,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
parallelThreadSize,
(LogEventConvert) binlogParser,
transactionBuffer,
destination);
destination, filterDmlInsert, filterDmlUpdate, filterDmlDelete);
mysqlMultiStageCoprocessor.setEventsPublishBlockingTime(eventsPublishBlockingTime);
return mysqlMultiStageCoprocessor;
}
@@ -224,6 +229,30 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
this.useDruidDdlFilter = useDruidDdlFilter;
}
public boolean isFilterDmlInsert() {
return filterDmlInsert;
}
public void setFilterDmlInsert(boolean filterDmlInsert) {
this.filterDmlInsert = filterDmlInsert;
}
public boolean isFilterDmlUpdate() {
return filterDmlUpdate;
}
public void setFilterDmlUpdate(boolean filterDmlUpdate) {
this.filterDmlUpdate = filterDmlUpdate;
}
public boolean isFilterDmlDelete() {
return filterDmlDelete;
}
public void setFilterDmlDelete(boolean filterDmlDelete) {
this.filterDmlDelete = filterDmlDelete;
}
public void setEnableTsdb(boolean enableTsdb) {
this.enableTsdb = enableTsdb;
if (this.enableTsdb) {

15
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
View File

@@ -72,13 +72,20 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
private BatchEventProcessor<MessageEvent> sinkStoreStage;
private LogContext logContext;
protected boolean filterDmlInsert = false;
protected boolean filterDmlUpdate = false;
protected boolean filterDmlDelete = false;
public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
EventTransactionBuffer transactionBuffer, String destination){
EventTransactionBuffer transactionBuffer, String destination, boolean filterDmlInsert, boolean filterDmlUpdate, boolean filterDmlDelete){
this.ringBufferSize = ringBufferSize;
this.parserThreadCount = parserThreadCount;
this.logEventConvert = logEventConvert;
this.transactionBuffer = transactionBuffer;
this.destination = destination;
this.filterDmlInsert = filterDmlInsert;
this.filterDmlUpdate = filterDmlUpdate;
this.filterDmlDelete = filterDmlDelete;
}
@Override
@@ -272,18 +279,18 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
tableMeta = logEventConvert.parseRowsEventForTableMeta((WriteRowsLogEvent) logEvent);
needDmlParse = true;
needDmlParse = !filterDmlInsert;//true;
break;
case LogEvent.UPDATE_ROWS_EVENT_V1:
case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
case LogEvent.UPDATE_ROWS_EVENT:
tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
needDmlParse = true;
needDmlParse = !filterDmlUpdate;//true;
break;
case LogEvent.DELETE_ROWS_EVENT_V1:
case LogEvent.DELETE_ROWS_EVENT:
tableMeta = logEventConvert.parseRowsEventForTableMeta((DeleteRowsLogEvent) logEvent);
needDmlParse = true;
needDmlParse = !filterDmlDelete;//true;
break;
case LogEvent.ROWS_QUERY_LOG_EVENT:
needDmlParse = true;