chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #726 , 优化SessionHandler的传输处理,提前序列化Entry

Browse Source
This commit is contained in:
七锋
2018-07-18 18:21:45 +08:00
parent 8e856acac5
commit 10bb800f85
18 changed files with 252 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.parse.inbound.mysql;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.locks.LockSupport;
import org.apache.commons.lang.StringUtils;
@@ -19,6 +20,7 @@ import com.lmax.disruptor.BlockingWaitStrategy;
import com.lmax.disruptor.EventFactory;
import com.lmax.disruptor.EventHandler;
import com.lmax.disruptor.ExceptionHandler;
import com.lmax.disruptor.InsufficientCapacityException;
import com.lmax.disruptor.LifecycleAware;
import com.lmax.disruptor.RingBuffer;
import com.lmax.disruptor.Sequence;
@@ -141,24 +143,35 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
* 网络数据投递
*/
public void publish(LogBuffer buffer) {
if (!isStart() && exception != null) {
throw exception;
}
long next = disruptorMsgBuffer.next();
MessageEvent event = disruptorMsgBuffer.get(next);
event.setBuffer(buffer);
disruptorMsgBuffer.publish(next);
publish(buffer, null);
}
public void publish(LogBuffer buffer, String binlogFileName) {
if (!isStart() && exception != null) {
throw exception;
}
long next = disruptorMsgBuffer.next();
MessageEvent event = disruptorMsgBuffer.get(next);
event.setBuffer(buffer);
event.setBinlogFileName(binlogFileName);
disruptorMsgBuffer.publish(next);
boolean interupted = false;
do {
try {
long next = disruptorMsgBuffer.tryNext();
MessageEvent event = disruptorMsgBuffer.get(next);
event.setBuffer(buffer);
if (binlogFileName != null) {
event.setBinlogFileName(binlogFileName);
}
disruptorMsgBuffer.publish(next);
break;
} catch (InsufficientCapacityException e) {
// park
LockSupport.parkNanos(1L);
interupted = Thread.interrupted();
}
} while (!interupted && isStart());
if (exception != null) {
throw exception;
}
}
@Override

4
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -70,7 +70,9 @@ public class DatabaseTableMeta implements TableMetaTSDB {
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "[scheduler-table-meta-snapshot]");
Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]");
thread.setDaemon(true);
return thread;
}
});

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -94,7 +94,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
tableMeta = tableMetas.get(keys);
if (tableMeta == null) {
Schema schemaRep = repository.findSchema(schema);
if (schema == null) {
if (schemaRep == null) {
return null;
}
SchemaObject data = schemaRep.findTable(table);

42
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java
View File

@@ -7,7 +7,6 @@ import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.store.CanalEventStore;
@@ -18,7 +17,7 @@ import com.alibaba.otter.canal.store.model.Events;
public class DummyEventStore implements CanalEventStore<Event> {
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static final String messgae = "{0} [{1}:{2}:{3}] {4} {5}.{6}";
private static final String messgae = "{0} [{1}:{2}:{3}]";
public void ack(Position position) throws CanalStoreException {
@@ -82,46 +81,40 @@ public class DummyEventStore implements CanalEventStore<Event> {
public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
Date date = new Date(data.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
new Object[] { Thread.currentThread().getName(), data.getJournalName(),
String.valueOf(data.getPosition()), format.format(date) }));
}
}
}
public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
Date date = new Date(data.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
new Object[] { Thread.currentThread().getName(), data.getJournalName(),
String.valueOf(data.getPosition()), format.format(date) }));
}
}
return true;
@@ -131,23 +124,20 @@ public class DummyEventStore implements CanalEventStore<Event> {
System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
for (Event data : datas) {
CanalEntry.Header header = data.getEntry().getHeader();
Date date = new Date(header.getExecuteTime());
Date date = new Date(data.getExecuteTime());
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
System.out.println(data.getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
new Object[] { Thread.currentThread().getName(), data.getJournalName(),
String.valueOf(data.getPosition()), format.format(date) }));
}
}