chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #449 , optimize findTransactionBeginPosition

Browse Source
This commit is contained in:
agapple
2017-12-27 17:34:46 +08:00
parent 9595e6abbd
commit 3104b4cc24
1 changed files with 21 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -8,7 +8,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TimerTask; import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -500,40 +499,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
// 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据 // 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition)
throws IOException { throws IOException {
// 尝试找到一个合适的位置
final AtomicBoolean reDump = new AtomicBoolean(false);
mysqlConnection.reconnect();
mysqlConnection.seek(entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction<LogEvent>() {
private LogPosition lastPosition;
public boolean sink(LogEvent event) {
try {
CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
if (entry == null) {
return true;
}
// 直接查询第一条业务数据，确认是否为事务Begin/End
if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType()
|| CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) {
lastPosition = buildLastPosition(entry);
return false;
} else {
reDump.set(true);
lastPosition = buildLastPosition(entry);
return false;
}
} catch (Exception e) {
// 上一次记录的poistion可能为一条update/insert/delete变更事件，直接进行dump的话，会缺少tableMap事件，导致tableId未进行解析
processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
reDump.set(true);
return false;
}
}
});
// 针对开始的第一条为非Begin记录，需要从该binlog扫描 // 针对开始的第一条为非Begin记录，需要从该binlog扫描
if (reDump.get()) {
final AtomicLong preTransactionStartPosition = new AtomicLong(0L); final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
mysqlConnection.reconnect(); mysqlConnection.reconnect();
mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() { mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() {
@@ -574,9 +540,6 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data"); throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
} }
return preTransactionStartPosition.get(); return preTransactionStartPosition.get();
} else {
return entryPosition.getPosition();
}
} }
// 根据时间查找binlog位置 // 根据时间查找binlog位置