chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #726 , 优化SessionHandler的传输处理,提前序列化Entry

Browse Source
This commit is contained in:
七锋
2018-07-18 18:21:45 +08:00
parent 8e856acac5
commit 10bb800f85
18 changed files with 252 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
store/src/main/java/com/alibaba/otter/canal/store/helper/CanalEventUtils.java
View File

@@ -2,7 +2,6 @@ package com.alibaba.otter.canal.store.helper;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.store.model.Event;
@@ -48,14 +47,13 @@ public class CanalEventUtils {
*/
public static LogPosition createPosition(Event event) {
EntryPosition position = new EntryPosition();
position.setJournalName(event.getEntry().getHeader().getLogfileName());
position.setPosition(event.getEntry().getHeader().getLogfileOffset());
position.setTimestamp(event.getEntry().getHeader().getExecuteTime());
position.setJournalName(event.getJournalName());
position.setPosition(event.getPosition());
position.setTimestamp(event.getExecuteTime());
// add serverId at 2016-06-28
position.setServerId(event.getEntry().getHeader().getServerId());
position.setServerId(event.getServerId());
// add gtid
position.setGtid(event.getEntry().getHeader().getGtid());
position.setGtid(event.getGtid());
LogPosition logPosition = new LogPosition();
logPosition.setPostion(position);
@@ -68,9 +66,9 @@ public class CanalEventUtils {
*/
public static LogPosition createPosition(Event event, boolean included) {
EntryPosition position = new EntryPosition();
position.setJournalName(event.getEntry().getHeader().getLogfileName());
position.setPosition(event.getEntry().getHeader().getLogfileOffset());
position.setTimestamp(event.getEntry().getHeader().getExecuteTime());
position.setJournalName(event.getJournalName());
position.setPosition(event.getPosition());
position.setTimestamp(event.getExecuteTime());
position.setIncluded(included);
LogPosition logPosition = new LogPosition();
@@ -84,13 +82,14 @@ public class CanalEventUtils {
*/
public static boolean checkPosition(Event event, LogPosition logPosition) {
EntryPosition position = logPosition.getPostion();
CanalEntry.Entry entry = event.getEntry();
boolean result = position.getTimestamp().equals(entry.getHeader().getExecuteTime());
boolean result = position.getTimestamp().equals(event.getExecuteTime());
boolean exactely = (StringUtils.isBlank(position.getJournalName()) && position.getPosition() == null);
if (!exactely) {// 精确匹配
result &= StringUtils.equals(entry.getHeader().getLogfileName(), position.getJournalName());
result &= position.getPosition().equals(entry.getHeader().getLogfileOffset());
result &= position.getPosition().equals(event.getPosition());
if (result) {// short path
result &= StringUtils.equals(event.getJournalName(), position.getJournalName());
}
}
return result;

11
store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java
View File

@@ -278,7 +278,7 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
// 提取数据并返回
for (; next <= end; next++) {
Event event = entries[getIndex(next)];
if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
if (ddlIsolation && isDdl(event.getEventType())) {
// 如果是ddl隔离，直接返回
if (entrys.size() == 0) {
entrys.add(event);// 如果没有DML事件，加入当前的DDL事件
@@ -297,7 +297,7 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
for (; memsize <= maxMemSize && next <= maxAbleSequence; next++) {
// 永远保证可以取出第一条的记录，避免死锁
Event event = entries[getIndex(next)];
if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
if (ddlIsolation && isDdl(event.getEventType())) {
// 如果是ddl隔离，直接返回
if (entrys.size() == 0) {
entrys.add(event);// 如果没有DML事件，加入当前的DDL事件
@@ -325,9 +325,8 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
for (int i = entrys.size() - 1; i >= 0; i--) {
Event event = entrys.get(i);
if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntry().getEntryType()
|| CanalEntry.EntryType.TRANSACTIONEND == event.getEntry().getEntryType()
|| isDdl(event.getEntry().getHeader().getEventType())) {
if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntryType()
|| CanalEntry.EntryType.TRANSACTIONEND == event.getEntryType() || isDdl(event.getEventType())) {
// 将事务头/尾设置可被为ack的点
range.setAck(CanalEventUtils.createPosition(event));
break;
@@ -532,7 +531,7 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
private long calculateSize(Event event) {
// 直接返回binlog中的事件大小
return event.getEntry().getHeader().getEventLength();
return event.getRawLength();
}
private int getIndex(long sequcnce) {

97
store/src/main/java/com/alibaba/otter/canal/store/model/Event.java
View File

@@ -6,7 +6,10 @@ import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.position.LogIdentity;
import com.google.protobuf.ByteString;
/**
* store存储数据对象
@@ -18,14 +21,32 @@ public class Event implements Serializable {
private static final long serialVersionUID = 1333330351758762739L;
private LogIdentity logIdentity; // 记录数据产生的来源
private CanalEntry.Entry entry;
private ByteString rawEntry;
private long executeTime;
private EntryType entryType;
private String journalName;
private long position;
private long serverId;
private EventType eventType;
private String gtid;
private long rawLength;
public Event(){
}
public Event(LogIdentity logIdentity, CanalEntry.Entry entry){
this.logIdentity = logIdentity;
this.entry = entry;
this.entryType = entry.getEntryType();
this.executeTime = entry.getHeader().getExecuteTime();
this.journalName = entry.getHeader().getLogfileName();
this.position = entry.getHeader().getLogfileOffset();
this.serverId = entry.getHeader().getServerId();
this.gtid = entry.getHeader().getGtid();
this.eventType = entry.getHeader().getEventType();
// build raw
this.rawEntry = entry.toByteString();
this.rawLength = rawEntry.size();
}
public LogIdentity getLogIdentity() {
@@ -36,12 +57,76 @@ public class Event implements Serializable {
this.logIdentity = logIdentity;
}
public CanalEntry.Entry getEntry() {
return entry;
public ByteString getRawEntry() {
return rawEntry;
}
public void setEntry(CanalEntry.Entry entry) {
this.entry = entry;
public void setRawEntry(ByteString rawEntry) {
this.rawEntry = rawEntry;
}
public long getExecuteTime() {
return executeTime;
}
public void setExecuteTime(long executeTime) {
this.executeTime = executeTime;
}
public EntryType getEntryType() {
return entryType;
}
public void setEntryType(EntryType entryType) {
this.entryType = entryType;
}
public String getJournalName() {
return journalName;
}
public void setJournalName(String journalName) {
this.journalName = journalName;
}
public long getPosition() {
return position;
}
public void setPosition(long position) {
this.position = position;
}
public long getServerId() {
return serverId;
}
public void setServerId(long serverId) {
this.serverId = serverId;
}
public String getGtid() {
return gtid;
}
public void setGtid(String gtid) {
this.gtid = gtid;
}
public long getRawLength() {
return rawLength;
}
public void setRawLength(long rawLength) {
this.rawLength = rawLength;
}
public EventType getEventType() {
return eventType;
}
public void setEventType(EventType eventType) {
this.eventType = eventType;
}
public String toString() {

7
store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java
View File

@@ -161,13 +161,12 @@ public class MemoryEventStoreMultiThreadTest extends MemoryEventStoreBase {
first = entrys.getPositionRange().getEnd();
for (Event event : entrys.getEvents()) {
this.result.add(event.getEntry().getHeader().getLogfileOffset());
this.result.add(event.getPosition());
}
emptyCount = 0;
System.out.println("offest : "
+ entrys.getEvents().get(0).getEntry().getHeader().getLogfileOffset()
+ " , count :" + entrys.getEvents().size());
System.out.println("offest : " + entrys.getEvents().get(0).getPosition() + " , count :"
+ entrys.getEvents().size());
ackCount++;
if (ackCount == 1) {
eventStore.cleanUntil(entrys.getPositionRange().getEnd());