chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #119 , support binlog row image noblob/minimal

Browse Source
This commit is contained in:
agapple
2015-06-29 23:52:37 +08:00
parent 21abbbfe47
commit 3d6174f857
12 changed files with 191 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -27,12 +27,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
public class MysqlConnection implements ErosaConnection {
private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class);
private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class);
private MysqlConnector connector;
private long slaveId;
private Charset charset = Charset.forName("UTF-8");
private BinlogFormat binlogFormat = BinlogFormat.ROW;
private Charset charset = Charset.forName("UTF-8");
private BinlogFormat binlogFormat;
private BinlogImage binlogImage;
public MysqlConnection(){
}
@@ -104,11 +105,6 @@ public class MysqlConnection implements ErosaConnection {
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
updateSettings();
BinlogFormat format = getBinlogFormat();
if (!format.isRow()) {
logger.warn("binlog_format : {} , it will not have rowData before/after columns", format.toString());
}
sendBinlogDump(binlogfilename, binlogPosition);
DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
fetcher.start(connector.getChannel());
@@ -212,7 +208,7 @@ public class MysqlConnection implements ErosaConnection {
}
/**
* 判断一下是否采用ROW模式
* 获取一下binlog format格式
*/
private void loadBinlogFormat() {
ResultSetPacket rs = null;
@@ -234,6 +230,30 @@ public class MysqlConnection implements ErosaConnection {
}
}
/**
* 获取一下binlog image格式
*/
private void loadBinlogImage() {
ResultSetPacket rs = null;
try {
rs = query("show variables like 'binlog_row_image'");
} catch (IOException e) {
throw new CanalParseException(e);
}
List<String> columnValues = rs.getFieldValues();
if (columnValues == null || columnValues.size() != 2) {
// 可能历时版本没有image特性
binlogImage = BinlogImage.FULL;
} else {
binlogImage = BinlogImage.valuesOf(columnValues.get(1));
}
if (binlogFormat == null) {
throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
}
}
public static enum BinlogFormat {
STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED");
@@ -267,6 +287,46 @@ public class MysqlConnection implements ErosaConnection {
}
}
/**
* http://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.
* html#sysvar_binlog_row_image
*
* @author agapple 2015年6月29日 下午10:39:03
* @since 1.0.20
*/
public static enum BinlogImage {
FULL("FULL"), MINIMAL("MINIMAL"), NOBLOB("NOBLOB");
public boolean isFull() {
return this == FULL;
}
public boolean isMinimal() {
return this == MINIMAL;
}
public boolean isNoBlob() {
return this == NOBLOB;
}
private String value;
private BinlogImage(String value){
this.value = value;
}
public static BinlogImage valuesOf(String value) {
BinlogImage[] formats = values();
for (BinlogImage format : formats) {
if (format.value.equalsIgnoreCase(value)) {
return format;
}
}
return null;
}
}
// ================== setter / getter ===================
public Charset getCharset() {
@@ -303,4 +363,14 @@ public class MysqlConnection implements ErosaConnection {
return binlogFormat;
}
public BinlogImage getBinlogImage() {
if (binlogImage == null) {
synchronized (this) {
loadBinlogImage();
}
}
return binlogImage;
}
}

58
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -22,6 +22,8 @@ import com.alibaba.otter.canal.parse.ha.CanalHAController;
import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback;
import com.alibaba.otter.canal.parse.inbound.SinkFunction;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogFormat;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogImage;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
@@ -62,6 +64,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private TableMetaCache tableMetaCache; // 对应meta
// cache
private int fallbackIntervalInSeconds = 60; // 切换回退时间
private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验
private BinlogImage[] supportBinlogImages; // 支持的binlogImage,如果设置会执行强校验
// 心跳检查
@@ -82,6 +86,34 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
throw new CanalParseException(e);
}
if (supportBinlogFormats != null && supportBinlogFormats.length > 0) {
BinlogFormat format = ((MysqlConnection) metaConnection).getBinlogFormat();
boolean found = false;
for (BinlogFormat supportFormat : supportBinlogFormats) {
if (supportFormat != null && format == supportFormat) {
found = true;
}
}
if (!found) {
throw new CanalParseException("Unsupported BinlogFormat " + format);
}
}
if (supportBinlogImages != null && supportBinlogImages.length > 0) {
BinlogImage image = ((MysqlConnection) metaConnection).getBinlogImage();
boolean found = false;
for (BinlogImage supportImage : supportBinlogImages) {
if (supportImage != null && image == supportImage) {
found = true;
}
}
if (!found) {
throw new CanalParseException("Unsupported BinlogImage " + image);
}
}
tableMetaCache = new TableMetaCache(metaConnection);
((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}
@@ -685,6 +717,32 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
return event;
}
public void setSupportBinlogFormats(String formatStrs) {
String[] formats = StringUtils.split(formatStrs, ',');
if (formats != null) {
BinlogFormat[] supportBinlogFormats = new BinlogFormat[formats.length];
int i = 0;
for (String format : formats) {
supportBinlogFormats[i++] = BinlogFormat.valuesOf(format);
}
this.supportBinlogFormats = supportBinlogFormats;
}
}
public void setSupportBinlogImages(String imageStrs) {
String[] images = StringUtils.split(imageStrs, ',');
if (images != null) {
BinlogImage[] supportBinlogImages = new BinlogImage[images.length];
int i = 0;
for (String image : images) {
supportBinlogImages[i++] = BinlogImage.valuesOf(image);
}
this.supportBinlogImages = supportBinlogImages;
}
}
// ===================== setter / getter ========================
public void setDefaultConnectionTimeoutInSeconds(int defaultConnectionTimeoutInSeconds) {

31
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -422,6 +422,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = columnInfo[i];
// mysql 5.6开始支持nolob/mininal类型,并不一定记录所有的列,需要进行判断
if (!cols.get(i)) {
continue;
}
Column.Builder columnBuilder = Column.newBuilder();
FieldMeta fieldMeta = null;
@@ -604,25 +609,21 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (index < 0) {
return false;
}
if ((bfColumns.size() - 1) < index) {
return false;
}
Column column = bfColumns.get(index);
if (column.getIsNull()) {
if (newValue != null) {
return true;
}
} else {
if (newValue == null) {
return true;
} else {
if (!column.getValue().equals(newValue)) {
return true;
for (Column column : bfColumns) {
if (column.getIndex() == index) {// 比较before / after的column index
if (column.getIsNull() && newValue == null) {
// 如果全是null
return false;
} else if (newValue != null && column.getValue().equals(newValue)) {
// 如果不围null，并且相等
return false;
}
}
}
return false;
// 比如nolob/minial模式下,可能找不到before记录,认为是有变化
return true;
}
private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {