chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #449 , optimize findTransactionBeginPosition

Browse Source
This commit is contained in:
agapple
2017-12-27 17:34:46 +08:00
parent 9595e6abbd
commit 3104b4cc24
1 changed files with 21 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -8,7 +8,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TimerTask; import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -500,10 +499,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
// 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据 // 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition)
throws IOException { throws IOException {
// 尝试找到一个合适的位置 // 针对开始的第一条为非Begin记录，需要从该binlog扫描
final AtomicBoolean reDump = new AtomicBoolean(false); final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
mysqlConnection.reconnect(); mysqlConnection.reconnect();
mysqlConnection.seek(entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction<LogEvent>() { mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() {
private LogPosition lastPosition; private LogPosition lastPosition;
@@ -514,69 +513,33 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
return true; return true;
} }
// 直接查询第一条业务数据，确认是否为事务Begin/End // 直接查询第一条业务数据，确认是否为事务Begin
if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType() // 记录一下transaction begin position
|| CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) { if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
lastPosition = buildLastPosition(entry); && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
return false; preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
} else {
reDump.set(true);
lastPosition = buildLastPosition(entry);
return false;
} }
if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
return false;// 退出
}
lastPosition = buildLastPosition(entry);
} catch (Exception e) { } catch (Exception e) {
// 上一次记录的poistion可能为一条update/insert/delete变更事件，直接进行dump的话，会缺少tableMap事件，导致tableId未进行解析
processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition()); processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
reDump.set(true);
return false; return false;
} }
return running;
} }
}); });
// 针对开始的第一条为非Begin记录，需要从该binlog扫描
if (reDump.get()) {
final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
mysqlConnection.reconnect();
mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() {
private LogPosition lastPosition; // 判断一下找到的最接近position的事务头的位置
if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
public boolean sink(LogEvent event) { logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
try { throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
if (entry == null) {
return true;
}
// 直接查询第一条业务数据，确认是否为事务Begin
// 记录一下transaction begin position
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
&& entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
}
if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
return false;// 退出
}
lastPosition = buildLastPosition(entry);
} catch (Exception e) {
processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
return false;
}
return running;
}
});
// 判断一下找到的最接近position的事务头的位置
if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
}
return preTransactionStartPosition.get();
} else {
return entryPosition.getPosition();
} }
return preTransactionStartPosition.get();
} }
// 根据时间查找binlog位置 // 根据时间查找binlog位置