chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge master branch.

Browse Source
This commit is contained in:
Chuanyi Li
2018-08-08 20:35:14 +08:00
parent 060bd8e31d 9e816bc48f
commit d4797a8fb9
34 changed files with 2857 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
View File

@@ -8,34 +8,38 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DefaultTableMetaTSDBFactory;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBBuilder;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBFactory;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
public abstract class AbstractMysqlEventParser extends AbstractEventParser {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected static final long BINLOG_START_OFFEST = 4L;
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected static final long BINLOG_START_OFFEST = 4L;
protected TableMetaTSDBFactory tableMetaTSDBFactory = new DefaultTableMetaTSDBFactory();
protected boolean enableTsdb = false;
protected String tsdbSpringXml;
protected TableMetaTSDB tableMetaTSDB;
protected boolean enableTsdb = false;
protected String tsdbSpringXml;
protected TableMetaTSDB tableMetaTSDB;
// 编码信息
protected byte connectionCharsetNumber = (byte) 33;
protected Charset connectionCharset = Charset.forName("UTF-8");
protected boolean filterQueryDcl = false;
protected boolean filterQueryDml = false;
protected boolean filterQueryDdl = false;
protected boolean filterRows = false;
protected boolean filterTableError = false;
protected boolean useDruidDdlFilter = true;
private final AtomicLong eventsPublishBlockingTime = new AtomicLong(0L);
protected byte connectionCharsetNumber = (byte) 33;
protected Charset connectionCharset = Charset.forName("UTF-8");
protected boolean filterQueryDcl = false;
protected boolean filterQueryDml = false;
protected boolean filterQueryDdl = false;
protected boolean filterRows = false;
protected boolean filterTableError = false;
protected boolean useDruidDdlFilter = true;
private final AtomicLong eventsPublishBlockingTime = new AtomicLong(0L);
protected BinlogParser buildParser() {
LogEventConvert convert = new LogEventConvert();
@@ -93,8 +97,16 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
public void start() throws CanalParseException {
if (enableTsdb) {
if (tableMetaTSDB == null) {
// 初始化
tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
synchronized (CanalEventParser.class) {
try {
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
System.setProperty("canal.instance.destination", destination);
// 初始化
tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml);
} finally {
System.setProperty("canal.instance.destination", "");
}
}
}
}
@@ -103,7 +115,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
public void stop() throws CanalParseException {
if (enableTsdb) {
TableMetaTSDBBuilder.destory(destination);
tableMetaTSDBFactory.destory(destination);
tableMetaTSDB = null;
}
@@ -177,7 +189,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
if (this.enableTsdb) {
if (tableMetaTSDB == null) {
// 初始化
tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml);
}
}
}
@@ -187,7 +199,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
if (this.enableTsdb) {
if (tableMetaTSDB == null) {
// 初始化
tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml);
}
}
}
@@ -195,5 +207,9 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
public AtomicLong getEventsPublishBlockingTime() {
return this.eventsPublishBlockingTime;
}
public void setTableMetaTSDBFactory(TableMetaTSDBFactory tableMetaTSDBFactory) {
this.tableMetaTSDBFactory = tableMetaTSDBFactory;
}
}

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -63,7 +63,6 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private String detectingSQL; // 心跳sql
private MysqlConnection metaConnection; // 查询meta信息的链接
private TableMetaCache tableMetaCache; // 对应meta
// cache
private int fallbackIntervalInSeconds = 60; // 切换回退时间
private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验
private BinlogImage[] supportBinlogImages; // 支持的binlogImage,如果设置会执行强校验

13
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -541,12 +541,16 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
}
rowsCount ++;
rowsCount++;
rowChangeBuider.addRowDatas(rowDataBuilder.build());
}
TableMapLogEvent table = event.getTable();
Header header = createHeader(event.getHeader(), table.getDbName(), table.getTableName(), eventType, rowsCount);
Header header = createHeader(event.getHeader(),
table.getDbName(),
table.getTableName(),
eventType,
rowsCount);
RowChange rowChange = rowChangeBuider.build();
if (tableError) {
@@ -801,12 +805,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType) {
return createHeader(logHeader, schemaName, tableName, eventType, -1);
}
private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType, Integer rowsCount) {
private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType,
Integer rowsCount) {
// header会做信息冗余,方便以后做检索或者过滤
Header.Builder headerBuilder = Header.newBuilder();
headerBuilder.setVersion(version);
@@ -960,5 +964,4 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
public void setGtidSet(GTIDSet gtidSet) {
this.gtidSet = gtidSet;
}
}

19
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DefaultTableMetaTSDBFactory.java Normal file
View File

@@ -0,0 +1,19 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
/**
* @author agapple 2017年10月11日 下午8:45:40
* @since 1.0.25
*/
public class DefaultTableMetaTSDBFactory implements TableMetaTSDBFactory {
/**
* 代理一下tableMetaTSDB的获取,使用隔离的spring定义
*/
public TableMetaTSDB build(String destination, String springXml) {
return TableMetaTSDBBuilder.build(destination, springXml);
}
public void destory(String destination) {
TableMetaTSDBBuilder.destory(destination);
}
}

7
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java
View File

@@ -10,12 +10,15 @@ import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.google.common.collect.Maps;
/**
* @author agapple 2017年10月11日 下午8:45:40
* tableMeta构造器
*
* @author agapple 2018年8月8日 上午11:01:08
* @since 1.0.25
*/
public class TableMetaTSDBBuilder {
protected final static Logger logger = LoggerFactory.getLogger(TableMetaTSDBBuilder.class);
protected final static Logger logger = LoggerFactory.getLogger(DefaultTableMetaTSDBFactory.class);
private static ConcurrentMap<String, ClassPathXmlApplicationContext> contexts = Maps.newConcurrentMap();
/**

18
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBFactory.java Normal file
View File

@@ -0,0 +1,18 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
/**
* tableMeta构造器,允许重载实现
*
* @author agapple 2018年8月8日 上午11:01:08
* @since 1.0.26
*/
public interface TableMetaTSDBFactory {
/**
* 代理一下tableMetaTSDB的获取,使用隔离的spring定义
*/
public TableMetaTSDB build(String destination, String springXml);
public void destory(String destination);
}

38
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java Normal file
View File

@@ -0,0 +1,38 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import org.junit.Test;
import java.net.InetSocketAddress;
import java.util.Date;
public class NoStorageTest {
final String DBNAME = "testdb";
final String TBNAME = "testtb";
final String DDL = "CREATE TABLE `testtb` (\n" +
" `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
" `name` varchar(2048) DEFAULT NULL,\n" +
" `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
" `otter_testcol` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
" PRIMARY KEY (`id`)\n" +
" ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
@Test
public void nostorage() {
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, null);
EntryPosition entryPosition = new EntryPosition();
entryPosition.setTimestamp(new Date().getTime());
String fullTableName = DBNAME + "." + TBNAME;
tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
entryPosition.setTimestamp(new Date().getTime() + 1000L);
TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
assert result.getDdl().equalsIgnoreCase(DDL);
}
}

77
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java Normal file
View File

@@ -0,0 +1,77 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import org.junit.Test;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class StorageTest {
final String DBNAME = "testdb";
final String TBNAME = "testtb";
final String DDL = "CREATE TABLE `testtb` (\n" +
" `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
" `name` varchar(2048) DEFAULT NULL,\n" +
" `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
" `otter_testcol` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
" `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
" PRIMARY KEY (`id`)\n" +
" ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
@Test
public void storage() {
MySqlTableMetaStorageFactory factory = new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
@Override
public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
}
@Override
public List<TableMetaEntry> fetch(String dbAddress, String dbName) {
TableMetaEntry tableMeta = new TableMetaEntry();
tableMeta.setSchema(DBNAME);
tableMeta.setTable(TBNAME);
tableMeta.setDdl(DDL);
tableMeta.setTimestamp(new Date().getTime());
List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
entries.add(tableMeta);
return entries;
}
@Override
public List<TableMetaEntry> fetch(String dbAddress, String dbName, String tableName) {
TableMetaEntry tableMeta = new TableMetaEntry();
tableMeta.setSchema(DBNAME);
tableMeta.setTable(TBNAME);
tableMeta.setDdl(DDL);
tableMeta.setTimestamp(new Date().getTime());
List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
entries.add(tableMeta);
return entries;
}
}, DBNAME);
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, factory.getTableMetaStorage());
EntryPosition entryPosition = new EntryPosition();
entryPosition.setTimestamp(new Date().getTime());
String fullTableName = DBNAME + "." + TBNAME;
tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
entryPosition.setTimestamp(new Date().getTime() + 1000L);
TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
assert result.getDdl().equalsIgnoreCase(DDL);
}
}