From 0c9eecd758a8cba0f218a121f83d3e8f01c4dee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=BB=BA=E5=AE=89?= Date: Thu, 29 Sep 2016 17:21:59 +0800 Subject: [PATCH 1/2] =?UTF-8?q?mysql5.6.29'@@global.binlog=5Fchecksum'?= =?UTF-8?q?=E5=B8=A6=E5=8D=95=E5=BC=95=E5=8F=B7=E5=AF=BC=E8=87=B4master?= =?UTF-8?q?=E9=80=80=E5=87=BA,=E4=BC=98=E5=8C=96binlogFormat=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/parse/inbound/mysql/MysqlConnection.java | 3 ++- .../otter/canal/parse/inbound/mysql/MysqlEventParser.java | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 5e19c1d4..2963df93 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -199,7 +199,8 @@ public class MysqlConnection implements ErosaConnection { // 如果不设置会出现错误： Slave can not handle replication events with the // checksum that master is configured to log // 但也不能乱设置，需要和mysql server的checksum配置一致，不然RotateLogEvent会出现乱码 - update("set @master_binlog_checksum= '@@global.binlog_checksum'"); + // '@@global.binlog_checksum'需要去掉单引号,在mysql 5.6.29下导致master退出 + update("set @master_binlog_checksum= @@global.binlog_checksum"); } catch (Exception e) { logger.warn(ExceptionUtils.getFullStackTrace(e)); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 548df70c..ca892a9f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -93,6 +93,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE for (BinlogFormat supportFormat : supportBinlogFormats) { if (supportFormat != null && format == supportFormat) { found = true; + break; } } @@ -107,6 +108,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE for (BinlogImage supportImage : supportBinlogImages) { if (supportImage != null && image == supportImage) { found = true; + break; } } From 96454d4387695459236ed988c88a453c61334e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=BB=BA=E5=AE=89?= Date: Mon, 10 Oct 2016 18:30:09 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=80=E5=90=AFchecksu?= =?UTF-8?q?m=E5=90=8ErotateLogEvent=E6=95=B0=E6=8D=AE=E5=8C=85=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E6=97=B6binlog=20name=E4=B9=B1=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../event/FormatDescriptionLogEvent.java | 9 +++++-- .../dbsync/binlog/event/RotateLogEvent.java | 1 + .../parse/inbound/AbstractEventParser.java | 1 - .../parse/inbound/mysql/MysqlConnection.java | 26 ++++++++++++++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java index 67fee9f0..279db972 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java @@ -140,10 +140,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 { throw new IOException("Unknown binlog version: " + binlogVersion); } } - + + public FormatDescriptionLogEvent(final int binlogVersion,int binlogChecksum){ + this(binlogVersion); + this.header.checksumAlg = binlogChecksum; + } + public FormatDescriptionLogEvent(final int binlogVersion){ this.binlogVersion = binlogVersion; - + postHeaderLen = new short[ENUM_END_EVENT]; /* identify binlog format */ switch (binlogVersion) { diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java index 7c058fd6..e79c8b0d 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java @@ -90,6 +90,7 @@ public final class RotateLogEvent extends LogEvent { int filenameLen = buffer.limit() - filenameOffset; if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1; buffer.position(filenameOffset); + filename = buffer.getFixString(filenameLen); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index 9e8c10ab..974632d8 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -325,7 +325,6 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle startTs = System.currentTimeMillis(); } CanalEntry.Entry event = binlogParser.parse(bod); - if (enabled) { this.parsingInterval = System.currentTimeMillis() - startTs; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 2963df93..a09ae326 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -7,6 +7,8 @@ import java.nio.charset.Charset; import java.util.List; import com.taobao.tddl.dbsync.binlog.LogPosition; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; + import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +37,8 @@ public class MysqlConnection implements ErosaConnection { private Charset charset = Charset.forName("UTF-8"); private BinlogFormat binlogFormat; private BinlogImage binlogImage; - + private int binlogChecksum; + public MysqlConnection(){ } @@ -107,12 +110,14 @@ public class MysqlConnection implements ErosaConnection { public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { updateSettings(); + loadBinlogChecksum(); sendBinlogDump(binlogfilename, binlogPosition); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); fetcher.start(connector.getChannel()); LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); LogContext context = new LogContext(); context.setLogPosition(new LogPosition(binlogfilename)); + context.setFormatDescription(new FormatDescriptionLogEvent(4,binlogChecksum)); while (fetcher.fetch()) { LogEvent event = null; event = decoder.decode(fetcher, context); @@ -259,7 +264,26 @@ public class MysqlConnection implements ErosaConnection { throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues()); } } + /** + * 获取主库checksum信息 + * https://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html#option_mysqld_binlog-checksum + */ + private void loadBinlogChecksum(){ + ResultSetPacket rs = null; + try { + rs = query("select @master_binlog_checksum"); + } catch (IOException e) { + throw new CanalParseException(e); + } + List columnValues = rs.getFieldValues(); + if(columnValues.get(0).toUpperCase().equals("CRC32")){ + binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32; + }else{ + binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF; + } + } + public static enum BinlogFormat { STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED");