chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
instance/core/pom.xml Normal file
View File

@@ -0,0 +1,35 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.19-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.instance.core</artifactId>
<packaging>jar</packaging>
<name>canal instance core module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.store</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.meta</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.parse</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

35
instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java Normal file
View File

@@ -0,0 +1,35 @@
package com.alibaba.otter.canal.instance.core;
import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.meta.CanalMetaManager;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.store.CanalEventStore;
/**
* 代表单个canal实例，比如一个destination会独立一个实例
*
* @author jianghang 2012-7-12 下午12:04:58
* @version 1.0.0
*/
public interface CanalInstance extends CanalLifeCycle {
public String getDestination();
public CanalEventParser getEventParser();
public CanalEventSink getEventSink();
public CanalEventStore getEventStore();
public CanalMetaManager getMetaManager();
public CanalAlarmHandler getAlarmHandler();
/**
* 客户端发生订阅/取消订阅行为
*/
public boolean subscribeChange(ClientIdentity identity);
}

16
instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceGenerator.java Normal file
View File

@@ -0,0 +1,16 @@
package com.alibaba.otter.canal.instance.core;
/**
* @author zebin.xuzb @ 2012-7-12
* @version 1.0.0
*/
public interface CanalInstanceGenerator {
/**
* 通过 destination 产生特定的 {@link CanalInstance}
*
* @param destination
* @return
*/
CanalInstance generate(String destination);
}

105
instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceSupport.java Normal file
View File

@@ -0,0 +1,105 @@
package com.alibaba.otter.canal.instance.core;
import java.util.List;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.ha.CanalHAController;
import com.alibaba.otter.canal.parse.ha.HeartBeatHAController;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
/**
* @author zebin.xuzb 2012-10-17 下午3:12:34
* @version 1.0.0
*/
public abstract class CanalInstanceSupport extends AbstractCanalLifeCycle {
protected void beforeStartEventParser(CanalEventParser eventParser) {
boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
startEventParserInternal(singleEventParser, true);
}
} else {
startEventParserInternal(eventParser, false);
}
}
// around event parser
protected void afterStartEventParser(CanalEventParser eventParser) {
// noop
}
// around event parser
protected void beforeStopEventParser(CanalEventParser eventParser) {
// noop
}
protected void afterStopEventParser(CanalEventParser eventParser) {
boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
stopEventParserInternal(singleEventParser);
}
} else {
stopEventParserInternal(eventParser);
}
}
/**
* 初始化单个eventParser，不需要考虑group
*/
protected void startEventParserInternal(CanalEventParser eventParser, boolean isGroup) {
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
// 首先启动log position管理器
CanalLogPositionManager logPositionManager = abstractEventParser.getLogPositionManager();
if (!logPositionManager.isStart()) {
logPositionManager.start();
}
}
if (eventParser instanceof MysqlEventParser) {
MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
CanalHAController haController = mysqlEventParser.getHaController();
if (haController instanceof HeartBeatHAController) {
((HeartBeatHAController) haController).setCanalHASwitchable(mysqlEventParser);
}
if (!haController.isStart()) {
haController.start();
}
}
}
protected void stopEventParserInternal(CanalEventParser eventParser) {
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
// 首先启动log position管理器
CanalLogPositionManager logPositionManager = abstractEventParser.getLogPositionManager();
if (logPositionManager.isStart()) {
logPositionManager.stop();
}
}
if (eventParser instanceof MysqlEventParser) {
MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
CanalHAController haController = mysqlEventParser.getHaController();
if (haController.isStart()) {
haController.stop();
}
}
}
}

20
instance/manager/pom.xml Normal file
View File

@@ -0,0 +1,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.19-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.instance.manager</artifactId>
<packaging>jar</packaging>
<name>canal instance manager module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.instance.core</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

29
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalConfigClient.java Normal file
View File

@@ -0,0 +1,29 @@
package com.alibaba.otter.canal.instance.manager;
import com.alibaba.otter.canal.instance.manager.model.Canal;
/**
* 对应canal的配置
*
* @author jianghang 2012-7-4 下午03:09:17
* @version 1.0.0
*/
public class CanalConfigClient {
/**
* 根据对应的destinantion查询Canal信息
*/
public Canal findCanal(String destination) {
// TODO 根据自己的业务实现
throw new UnsupportedOperationException();
}
/**
* 根据对应的destinantion查询filter信息
*/
public String findFilter(String destination) {
// TODO 根据自己的业务实现
throw new UnsupportedOperationException();
}
}

550
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java Normal file
View File

@@ -0,0 +1,550 @@
package com.alibaba.otter.canal.instance.manager;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.common.alarm.LogAlarmHandler;
import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceSupport;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode;
import com.alibaba.otter.canal.meta.CanalMetaManager;
import com.alibaba.otter.canal.meta.MemoryMetaManager;
import com.alibaba.otter.canal.meta.PeriodMixedMetaManager;
import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.ha.CanalHAController;
import com.alibaba.otter.canal.parse.ha.HeartBeatHAController;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
import com.alibaba.otter.canal.store.AbstractCanalStoreScavenge;
import com.alibaba.otter.canal.store.CanalEventStore;
import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
import com.alibaba.otter.canal.store.model.BatchMode;
import com.alibaba.otter.canal.store.model.Event;
/**
* 单个canal实例，比如一个destination会独立一个实例
*
* @author jianghang 2012-7-11 下午09:26:51
* @version 1.0.0
*/
public class CanalInstanceWithManager extends CanalInstanceSupport implements CanalInstance {
private static final Logger logger = LoggerFactory.getLogger(CanalInstanceWithManager.class);
protected Long canalId; // 和manager交互唯一标示
protected String destination; // 队列名字
protected String filter; // 过滤表达式
protected CanalParameter parameters; // 对应参数
protected CanalMetaManager metaManager; // 消费信息管理器
protected CanalEventStore<Event> eventStore; // 有序队列
protected CanalEventParser eventParser; // 解析对应的数据信息
protected CanalEventSink<List<Entry>> eventSink; // 链接parse和store的桥接器
protected CanalAlarmHandler alarmHandler; // alarm报警机制
protected ZkClientx zkClientx;
public CanalInstanceWithManager(Canal canal){
this(canal, null);
}
public CanalInstanceWithManager(Canal canal, String filter){
this.parameters = canal.getCanalParameter();
this.canalId = canal.getId();
this.destination = canal.getName();
this.filter = filter;
logger.info("init CannalInstance for {}-{} with parameters:{}",
new Object[] { canalId, destination, parameters });
// 初始化报警机制
initAlarmHandler();
// 初始化metaManager
initMetaManager();
// 初始化eventStore
initEventStore();
// 初始化eventSink
initEventSink();
// 初始化eventParser;
initEventParser();
// 基础工具，需要提前start，会有先订阅再根据filter条件启动paser的需求
if (!alarmHandler.isStart()) {
alarmHandler.start();
}
if (!metaManager.isStart()) {
metaManager.start();
}
logger.info("init successful....");
}
public void start() {
super.start();
// 初始化metaManager
logger.info("start CannalInstance for {}-{} with parameters:{}", new Object[] { canalId, destination,
parameters });
if (!metaManager.isStart()) {
metaManager.start();
}
if (!alarmHandler.isStart()) {
alarmHandler.start();
}
if (!eventStore.isStart()) {
eventStore.start();
}
if (!eventSink.isStart()) {
eventSink.start();
}
if (!eventParser.isStart()) {
beforeStartEventParser(eventParser);
eventParser.start();
}
logger.info("start successful....");
}
public void stop() {
logger.info("stop CannalInstance for {}-{} ", new Object[] { canalId, destination });
if (eventParser.isStart()) {
eventParser.stop();
afterStopEventParser(eventParser);
}
if (eventSink.isStart()) {
eventSink.stop();
}
if (eventStore.isStart()) {
eventStore.stop();
}
if (metaManager.isStart()) {
metaManager.stop();
}
if (alarmHandler.isStart()) {
alarmHandler.stop();
}
// if (zkClientx != null) {
// zkClientx.close();
// }
super.stop();
logger.info("stop successful....");
}
public boolean subscribeChange(ClientIdentity identity) {
if (StringUtils.isNotEmpty(identity.getFilter())) {
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());
boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
}
} else {
((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
}
}
// filter的处理规则
// a. parser处理数据过滤处理
// b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
// 后续内存版的一对多分发，可以考虑
return true;
}
protected void afterStartEventParser(CanalEventParser eventParser) {
super.afterStartEventParser(eventParser);
// 读取一下历史订阅的filter信息
List<ClientIdentity> clientIdentitys = metaManager.listAllSubscribeInfo(destination);
for (ClientIdentity clientIdentity : clientIdentitys) {
subscribeChange(clientIdentity);
}
}
protected void initAlarmHandler() {
logger.info("init alarmHandler begin...");
alarmHandler = new LogAlarmHandler();
logger.info("init alarmHandler end! \n\t load CanalAlarmHandler:{} ", alarmHandler.getClass().getName());
}
protected void initMetaManager() {
logger.info("init metaManager begin...");
MetaMode mode = parameters.getMetaMode();
if (mode.isMemory()) {
metaManager = new MemoryMetaManager();
} else if (mode.isZookeeper()) {
metaManager = new ZooKeeperMetaManager();
((ZooKeeperMetaManager) metaManager).setZkClientx(getZkclientx());
} else if (mode.isMixed()) {
// metaManager = new MixedMetaManager();
metaManager = new PeriodMixedMetaManager();// 换用优化过的mixed, at
// 2012-09-11
// 设置内嵌的zk metaManager
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(getZkclientx());
((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
} else {
throw new CanalException("unsupport MetaMode for " + mode);
}
logger.info("init metaManager end! \n\t load CanalMetaManager:{} ", metaManager.getClass().getName());
}
protected void initEventStore() {
logger.info("init eventStore begin...");
StorageMode mode = parameters.getStorageMode();
if (mode.isMemory()) {
MemoryEventStoreWithBuffer memoryEventStore = new MemoryEventStoreWithBuffer();
memoryEventStore.setBufferSize(parameters.getMemoryStorageBufferSize());
memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
eventStore = memoryEventStore;
} else if (mode.isFile()) {
// 后续版本支持
throw new CanalException("unsupport MetaMode for " + mode);
} else if (mode.isMixed()) {
// 后续版本支持
throw new CanalException("unsupport MetaMode for " + mode);
} else {
throw new CanalException("unsupport MetaMode for " + mode);
}
if (eventStore instanceof AbstractCanalStoreScavenge) {
StorageScavengeMode scavengeMode = parameters.getStorageScavengeMode();
AbstractCanalStoreScavenge eventScavengeStore = (AbstractCanalStoreScavenge) eventStore;
eventScavengeStore.setDestination(destination);
eventScavengeStore.setCanalMetaManager(metaManager);
eventScavengeStore.setOnAck(scavengeMode.isOnAck());
eventScavengeStore.setOnFull(scavengeMode.isOnFull());
eventScavengeStore.setOnSchedule(scavengeMode.isOnSchedule());
if (scavengeMode.isOnSchedule()) {
eventScavengeStore.setScavengeSchedule(parameters.getScavengeSchdule());
}
}
logger.info("init eventStore end! \n\t load CanalEventStore:{}", eventStore.getClass().getName());
}
protected void initEventSink() {
logger.info("init eventSink begin...");
int groupSize = getGroupSize();
if (groupSize <= 1) {
eventSink = new EntryEventSink();
} else {
eventSink = new GroupEventSink(groupSize);
}
if (eventSink instanceof EntryEventSink) {
((EntryEventSink) eventSink).setFilterTransactionEntry(false);
((EntryEventSink) eventSink).setEventStore(getEventStore());
}
// if (StringUtils.isNotEmpty(filter)) {
// AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
// ((AbstractCanalEventSink) eventSink).setFilter(aviaterFilter);
// }
logger.info("init eventSink end! \n\t load CanalEventSink:{}", eventSink.getClass().getName());
}
protected void initEventParser() {
logger.info("init eventParser begin...");
SourcingType type = parameters.getSourcingType();
List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
if (!CollectionUtils.isEmpty(groupDbAddresses)) {
int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
for (int i = 0; i < size; i++) {
List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>();
SourcingType lastType = null;
for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
lastType,
groupDbAddress.get(i).getType()));
}
lastType = groupDbAddress.get(i).getType();
dbAddress.add(groupDbAddress.get(i).getDbAddress());
}
// 初始化其中的一个分组parser
eventParsers.add(doInitEventParser(lastType, dbAddress));
}
if (eventParsers.size() > 1) { // 如果存在分组，构造分组的parser
GroupEventParser groupEventParser = new GroupEventParser();
groupEventParser.setEventParsers(eventParsers);
this.eventParser = groupEventParser;
} else {
this.eventParser = eventParsers.get(0);
}
} else {
// 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>());
}
logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}
private CanalEventParser doInitEventParser(SourcingType type, List<InetSocketAddress> dbAddresses) {
CanalEventParser eventParser = null;
if (type.isMysql()) {
MysqlEventParser mysqlEventParser = new MysqlEventParser();
mysqlEventParser.setDestination(destination);
// 编码参数
mysqlEventParser.setConnectionCharset(Charset.forName(parameters.getConnectionCharset()));
mysqlEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
// 网络相关参数
mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
// 心跳检查参数
mysqlEventParser.setDetectingEnable(parameters.getDetectingEnable());
mysqlEventParser.setDetectingSQL(parameters.getDetectingSQL());
mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
// 数据库信息参数
mysqlEventParser.setSlaveId(parameters.getSlaveId());
if (!CollectionUtils.isEmpty(dbAddresses)) {
mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
parameters.getDbUsername(),
parameters.getDbPassword(),
parameters.getDefaultDatabaseName()));
if (dbAddresses.size() > 1) {
mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1),
parameters.getDbUsername(),
parameters.getDbPassword(),
parameters.getDefaultDatabaseName()));
}
}
if (!CollectionUtils.isEmpty(parameters.getPositions())) {
EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
EntryPosition.class);
// binlog位置参数
mysqlEventParser.setMasterPosition(masterPosition);
if (parameters.getPositions().size() > 1) {
EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
EntryPosition.class);
mysqlEventParser.setStandbyPosition(standbyPosition);
}
}
mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
mysqlEventParser.setProfilingEnabled(false);
mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
eventParser = mysqlEventParser;
} else if (type.isLocalBinlog()) {
LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
localBinlogEventParser.setDestination(destination);
localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
localBinlogEventParser.setConnectionCharset(Charset.forName(parameters.getConnectionCharset()));
localBinlogEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
localBinlogEventParser.setProfilingEnabled(false);
localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
// 数据库信息，反查表结构时需要
if (!CollectionUtils.isEmpty(dbAddresses)) {
localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
parameters.getDbUsername(),
parameters.getDbPassword(),
parameters.getDefaultDatabaseName()));
}
eventParser = localBinlogEventParser;
} else if (type.isOracle()) {
throw new CanalException("unsupport SourcingType for " + type);
} else {
throw new CanalException("unsupport SourcingType for " + type);
}
// add transaction support at 2012-12-06
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
abstractEventParser.setTransactionSize(parameters.getTransactionSize());
abstractEventParser.setLogPositionManager(initLogPositionManager());
abstractEventParser.setAlarmHandler(getAlarmHandler());
abstractEventParser.setEventSink(getEventSink());
if (StringUtils.isNotEmpty(filter)) {
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
abstractEventParser.setEventFilter(aviaterFilter);
}
// 设置黑名单
if (StringUtils.isNotEmpty(parameters.getBlackFilter())) {
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(parameters.getBlackFilter());
abstractEventParser.setEventBlackFilter(aviaterFilter);
}
}
if (eventParser instanceof MysqlEventParser) {
MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
// 初始化haController，绑定与eventParser的关系，haController会控制eventParser
CanalHAController haController = initHaController();
mysqlEventParser.setHaController(haController);
}
return eventParser;
}
protected CanalHAController initHaController() {
logger.info("init haController begin...");
HAMode haMode = parameters.getHaMode();
CanalHAController haController = null;
if (haMode.isHeartBeat()) {
haController = new HeartBeatHAController();
((HeartBeatHAController) haController).setDetectingRetryTimes(parameters.getDetectingRetryTimes());
((HeartBeatHAController) haController).setSwitchEnable(parameters.getHeartbeatHaEnable());
} else {
throw new CanalException("unsupport HAMode for " + haMode);
}
logger.info("init haController end! \n\t load CanalHAController:{}", haController.getClass().getName());
return haController;
}
protected CanalLogPositionManager initLogPositionManager() {
logger.info("init logPositionPersistManager begin...");
IndexMode indexMode = parameters.getIndexMode();
CanalLogPositionManager logPositionManager = null;
if (indexMode.isMemory()) {
logPositionManager = new MemoryLogPositionManager();
} else if (indexMode.isZookeeper()) {
logPositionManager = new ZooKeeperLogPositionManager();
((ZooKeeperLogPositionManager) logPositionManager).setZkClientx(getZkclientx());
} else if (indexMode.isMixed()) {
logPositionManager = new PeriodMixedLogPositionManager();
ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager();
zooKeeperLogPositionManager.setZkClientx(getZkclientx());
((PeriodMixedLogPositionManager) logPositionManager).setZooKeeperLogPositionManager(zooKeeperLogPositionManager);
} else if (indexMode.isMeta()) {
logPositionManager = new MetaLogPositionManager();
((MetaLogPositionManager) logPositionManager).setMetaManager(metaManager);
} else if (indexMode.isMemoryMetaFailback()) {
MemoryLogPositionManager primaryLogPositionManager = new MemoryLogPositionManager();
MetaLogPositionManager failbackLogPositionManager = new MetaLogPositionManager();
failbackLogPositionManager.setMetaManager(metaManager);
logPositionManager = new FailbackLogPositionManager();
((FailbackLogPositionManager) logPositionManager).setPrimary(primaryLogPositionManager);
((FailbackLogPositionManager) logPositionManager).setFailback(failbackLogPositionManager);
} else {
throw new CanalException("unsupport indexMode for " + indexMode);
}
logger.info("init logPositionManager end! \n\t load CanalLogPositionManager:{}", logPositionManager.getClass()
.getName());
return logPositionManager;
}
protected void startEventParserInternal(CanalEventParser eventParser, boolean isGroup) {
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
abstractEventParser.setAlarmHandler(getAlarmHandler());
}
super.startEventParserInternal(eventParser, isGroup);
}
private int getGroupSize() {
List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
if (!CollectionUtils.isEmpty(groupDbAddresses)) {
return groupDbAddresses.get(0).size();
} else {
// 可能是基于tddl的启动
return 1;
}
}
private synchronized ZkClientx getZkclientx() {
// 做一下排序，保证相同的机器只使用同一个链接
List<String> zkClusters = new ArrayList<String>(parameters.getZkClusters());
Collections.sort(zkClusters);
return ZkClientx.getZkClient(StringUtils.join(zkClusters, ";"));
}
// =====================================
public String getDestination() {
return destination;
}
public CanalMetaManager getMetaManager() {
return metaManager;
}
public CanalEventStore<Event> getEventStore() {
return eventStore;
}
public CanalEventParser getEventParser() {
return eventParser;
}
public CanalEventSink<List<Entry>> getEventSink() {
return eventSink;
}
public CanalAlarmHandler getAlarmHandler() {
return alarmHandler;
}
public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
this.alarmHandler = alarmHandler;
}
}

29
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/ManagerCanalInstanceGenerator.java Normal file
View File

@@ -0,0 +1,29 @@
package com.alibaba.otter.canal.instance.manager;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.model.Canal;
/**
* 基于manager生成对应的{@linkplain CanalInstance}
*
* @author jianghang 2012-7-12 下午05:37:09
* @version 1.0.0
*/
public class ManagerCanalInstanceGenerator implements CanalInstanceGenerator {
private CanalConfigClient canalConfigClient;
public CanalInstance generate(String destination) {
Canal canal = canalConfigClient.findCanal(destination);
String filter = canalConfigClient.findFilter(destination);
return new CanalInstanceWithManager(canal, filter);
}
// ================ setter / getter ================
public void setCanalConfigClient(CanalConfigClient canalConfigClient) {
this.canalConfigClient = canalConfigClient;
}
}

88
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/Canal.java Normal file
View File

@@ -0,0 +1,88 @@
package com.alibaba.otter.canal.instance.manager.model;
import java.io.Serializable;
import java.util.Date;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* 对应的canal模型对象
*
* @author jianghang 2012-7-4 下午02:32:39
* @version 1.0.0
*/
public class Canal implements Serializable {
private static final long serialVersionUID = 8333284022624682754L;
private Long id;
private String name; // 对应的名字
private String desc; // 描述
private CanalStatus status;
private CanalParameter canalParameter; // 参数定义
private Date gmtCreate; // 创建时间
private Date gmtModified; // 修改时间
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDesc() {
return desc;
}
public void setDesc(String desc) {
this.desc = desc;
}
public CanalParameter getCanalParameter() {
return canalParameter;
}
public void setCanalParameter(CanalParameter canalParameter) {
this.canalParameter = canalParameter;
}
public CanalStatus getStatus() {
return status;
}
public void setStatus(CanalStatus status) {
this.status = status;
}
public Date getGmtCreate() {
return gmtCreate;
}
public void setGmtCreate(Date gmtCreate) {
this.gmtCreate = gmtCreate;
}
public Date getGmtModified() {
return gmtModified;
}
public void setGmtModified(Date gmtModified) {
this.gmtModified = gmtModified;
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

865
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java Normal file
View File

@@ -0,0 +1,865 @@
package com.alibaba.otter.canal.instance.manager.model;
import java.io.Serializable;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
/**
* canal运行相关参数
*
* @author jianghang 2012-7-4 下午02:52:52
* @version 1.0.0
*/
public class CanalParameter implements Serializable {
private static final long serialVersionUID = -5893459662315430900L;
private Long canalId;
// 相关参数
private RunMode runMode = RunMode.EMBEDDED; // 运行模式：嵌入式/服务式
private ClusterMode clusterMode = ClusterMode.STANDALONE; // 集群模式：单机/冷备/热备份
private Long zkClusterId; // zk集群id，为管理方便
private List<String> zkClusters; // zk集群地址
// meta相关参数
private MetaMode metaMode = MetaMode.MEMORY; // meta机制
// storage存储
private Integer transactionSize = 1024; // 支持处理的transaction事务大小
private StorageMode storageMode = StorageMode.MEMORY; // 存储机制
private BatchMode storageBatchMode = BatchMode.MEMSIZE; // 基于大小返回结果
private Integer memoryStorageBufferSize = 16 * 1024; // 内存存储的buffer大小
private Integer memoryStorageBufferMemUnit = 1024; // 内存存储的buffer内存占用单位，默认为1kb
private String fileStorageDirectory; // 文件存储的目录位置
private Integer fileStorageStoreCount; // 每个文件store存储的记录数
private Integer fileStorageRollverCount; // store文件的个数
private Integer fileStoragePercentThresold; // 整个store存储占disk硬盘的百分比，超过百分比及时条数还未满也不写入
private StorageScavengeMode storageScavengeMode = StorageScavengeMode.ON_ACK;
private String scavengeSchdule; // 调度规则
// replcation相关参数
private SourcingType sourcingType = SourcingType.MYSQL; // 数据来源类型
private String localBinlogDirectory; // 本地localBinlog目录
private HAMode haMode = HAMode.HEARTBEAT; // ha机制
// 网络链接参数
private Integer port = 11111; // 服务端口，独立运行时需要配置
private Integer defaultConnectionTimeoutInSeconds = 30; // sotimeout
private Integer receiveBufferSize = 64 * 1024;
private Integer sendBufferSize = 64 * 1024;
// 编码信息
private Byte connectionCharsetNumber = (byte) 33;
private String connectionCharset = "UTF-8";
// 数据库信息
private List<InetSocketAddress> dbAddresses; // 数据库链接信息
private List<List<DataSourcing>> groupDbAddresses; // 数据库链接信息，包含多组信息
private String dbUsername; // 数据库用户
private String dbPassword; // 数据库密码
// binlog链接信息
private IndexMode indexMode;
private List<String> positions; // 数据库positions信息
private String defaultDatabaseName; // 默认链接的数据库schmea
private Long slaveId; // 链接到mysql的slaveId
private Integer fallbackIntervalInSeconds = 60; // 数据库发生切换查找时回退的时间
// 心跳检查信息
private Boolean detectingEnable = true; // 是否开启心跳语句
private Boolean heartbeatHaEnable = false; // 是否开启基于心跳检查的ha功能
private String detectingSQL; // 心跳sql
private Integer detectingIntervalInSeconds = 3; // 检测频率
private Integer detectingTimeoutThresholdInSeconds = 30; // 心跳超时时间
private Integer detectingRetryTimes = 3; // 心跳检查重试次数
// tddl/diamond 配置信息
private String app;
private String group;
// media配置信息
private String mediaGroup;
// metaq 存储配置信息
private String metaqStoreUri;
// ddl同步支持，隔离dml/ddl
private Boolean ddlIsolation = Boolean.FALSE; // 是否将ddl单条返回
private Boolean filterTableError = Boolean.FALSE; // 是否忽略表解析异常
private String blackFilter = null; // 匹配黑名单,忽略解析
// ================================== 兼容字段处理
private InetSocketAddress masterAddress; // 主库信息
private String masterUsername; // 帐号
private String masterPassword; // 密码
private InetSocketAddress standbyAddress; // 备库信息
private String standbyUsername; // 帐号
private String standbyPassword;
private String masterLogfileName = null; // master起始位置
private Long masterLogfileOffest = null;
private Long masterTimestamp = null;
private String standbyLogfileName = null; // standby起始位置
private Long standbyLogfileOffest = null;
private Long standbyTimestamp = null;
public static enum RunMode {
/** 嵌入式 */
EMBEDDED,
/** 服务式 */
SERVICE;
public boolean isEmbedded() {
return this.equals(RunMode.EMBEDDED);
}
public boolean isService() {
return this.equals(RunMode.SERVICE);
}
}
public static enum ClusterMode {
/** 嵌入式 */
STANDALONE,
/** 冷备 */
STANDBY,
/** 热备 */
ACTIVE;
public boolean isStandalone() {
return this.equals(ClusterMode.STANDALONE);
}
public boolean isStandby() {
return this.equals(ClusterMode.STANDBY);
}
public boolean isActive() {
return this.equals(ClusterMode.ACTIVE);
}
}
public static enum HAMode {
/** 心跳检测 */
HEARTBEAT,
/** otter media */
MEDIA;
public boolean isHeartBeat() {
return this.equals(HAMode.HEARTBEAT);
}
public boolean isMedia() {
return this.equals(HAMode.MEDIA);
}
}
public static enum StorageMode {
/** 内存存储模式 */
MEMORY,
/** 文件存储模式 */
FILE,
/** 混合模式，内存+文件 */
MIXED;
public boolean isMemory() {
return this.equals(StorageMode.MEMORY);
}
public boolean isFile() {
return this.equals(StorageMode.FILE);
}
public boolean isMixed() {
return this.equals(StorageMode.MIXED);
}
}
public static enum StorageScavengeMode {
/** 在存储满的时候触发 */
ON_FULL,
/** 在每次有ack请求时触发 */
ON_ACK,
/** 定时触发，需要外部控制 */
ON_SCHEDULE,
/** 不做任何操作，由外部进行清理 */
NO_OP;
public boolean isOnFull() {
return this.equals(StorageScavengeMode.ON_FULL);
}
public boolean isOnAck() {
return this.equals(StorageScavengeMode.ON_ACK);
}
public boolean isOnSchedule() {
return this.equals(StorageScavengeMode.ON_SCHEDULE);
}
public boolean isNoop() {
return this.equals(StorageScavengeMode.NO_OP);
}
}
public static enum SourcingType {
/** mysql DB */
MYSQL,
/** localBinLog */
LOCALBINLOG,
/** oracle DB */
ORACLE,
/** 多库合并模式 */
GROUP;
public boolean isMysql() {
return this.equals(SourcingType.MYSQL);
}
public boolean isLocalBinlog() {
return this.equals(SourcingType.LOCALBINLOG);
}
public boolean isOracle() {
return this.equals(SourcingType.ORACLE);
}
public boolean isGroup() {
return this.equals(SourcingType.GROUP);
}
}
public static enum MetaMode {
/** 内存存储模式 */
MEMORY,
/** 文件存储模式 */
ZOOKEEPER,
/** 混合模式，内存+文件 */
MIXED;
public boolean isMemory() {
return this.equals(MetaMode.MEMORY);
}
public boolean isZookeeper() {
return this.equals(MetaMode.ZOOKEEPER);
}
public boolean isMixed() {
return this.equals(MetaMode.MIXED);
}
}
public static enum IndexMode {
/** 内存存储模式 */
MEMORY,
/** 文件存储模式 */
ZOOKEEPER,
/** 混合模式，内存+文件 */
MIXED,
/** 基于meta信息 */
META,
/** 基于内存+meta的failback实现 */
MEMORY_META_FAILBACK;
public boolean isMemory() {
return this.equals(IndexMode.MEMORY);
}
public boolean isZookeeper() {
return this.equals(IndexMode.ZOOKEEPER);
}
public boolean isMixed() {
return this.equals(IndexMode.MIXED);
}
public boolean isMeta() {
return this.equals(IndexMode.META);
}
public boolean isMemoryMetaFailback() {
return this.equals(IndexMode.MEMORY_META_FAILBACK);
}
}
public static enum BatchMode {
/** 对象数量 */
ITEMSIZE,
/** 内存大小 */
MEMSIZE;
public boolean isItemSize() {
return this == BatchMode.ITEMSIZE;
}
public boolean isMemSize() {
return this == BatchMode.MEMSIZE;
}
}
/**
* 数据来源描述
*
* @author jianghang 2012-12-26 上午11:05:20
* @version 4.1.5
*/
public static class DataSourcing implements Serializable {
private static final long serialVersionUID = -1770648468678085234L;
private SourcingType type;
private InetSocketAddress dbAddress;
public DataSourcing(){
}
public DataSourcing(SourcingType type, InetSocketAddress dbAddress){
this.type = type;
this.dbAddress = dbAddress;
}
public SourcingType getType() {
return type;
}
public void setType(SourcingType type) {
this.type = type;
}
public InetSocketAddress getDbAddress() {
return dbAddress;
}
public void setDbAddress(InetSocketAddress dbAddress) {
this.dbAddress = dbAddress;
}
}
public Long getCanalId() {
return canalId;
}
public void setCanalId(Long canalId) {
this.canalId = canalId;
}
public RunMode getRunMode() {
return runMode;
}
public void setRunMode(RunMode runMode) {
this.runMode = runMode;
}
public ClusterMode getClusterMode() {
return clusterMode;
}
public void setClusterMode(ClusterMode clusterMode) {
this.clusterMode = clusterMode;
}
public List<String> getZkClusters() {
return zkClusters;
}
public void setZkClusters(List<String> zkClusters) {
this.zkClusters = zkClusters;
}
public MetaMode getMetaMode() {
return metaMode;
}
public void setMetaMode(MetaMode metaMode) {
this.metaMode = metaMode;
}
public StorageMode getStorageMode() {
return storageMode;
}
public void setStorageMode(StorageMode storageMode) {
this.storageMode = storageMode;
}
public Integer getMemoryStorageBufferSize() {
return memoryStorageBufferSize;
}
public void setMemoryStorageBufferSize(Integer memoryStorageBufferSize) {
this.memoryStorageBufferSize = memoryStorageBufferSize;
}
public String getFileStorageDirectory() {
return fileStorageDirectory;
}
public void setFileStorageDirectory(String fileStorageDirectory) {
this.fileStorageDirectory = fileStorageDirectory;
}
public Integer getFileStorageStoreCount() {
return fileStorageStoreCount;
}
public void setFileStorageStoreCount(Integer fileStorageStoreCount) {
this.fileStorageStoreCount = fileStorageStoreCount;
}
public Integer getFileStorageRollverCount() {
return fileStorageRollverCount;
}
public void setFileStorageRollverCount(Integer fileStorageRollverCount) {
this.fileStorageRollverCount = fileStorageRollverCount;
}
public Integer getFileStoragePercentThresold() {
return fileStoragePercentThresold;
}
public void setFileStoragePercentThresold(Integer fileStoragePercentThresold) {
this.fileStoragePercentThresold = fileStoragePercentThresold;
}
public SourcingType getSourcingType() {
return sourcingType;
}
public void setSourcingType(SourcingType sourcingType) {
this.sourcingType = sourcingType;
}
public String getLocalBinlogDirectory() {
return localBinlogDirectory;
}
public void setLocalBinlogDirectory(String localBinlogDirectory) {
this.localBinlogDirectory = localBinlogDirectory;
}
public HAMode getHaMode() {
return haMode;
}
public void setHaMode(HAMode haMode) {
this.haMode = haMode;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public Integer getDefaultConnectionTimeoutInSeconds() {
return defaultConnectionTimeoutInSeconds;
}
public void setDefaultConnectionTimeoutInSeconds(Integer defaultConnectionTimeoutInSeconds) {
this.defaultConnectionTimeoutInSeconds = defaultConnectionTimeoutInSeconds;
}
public Integer getReceiveBufferSize() {
return receiveBufferSize;
}
public void setReceiveBufferSize(Integer receiveBufferSize) {
this.receiveBufferSize = receiveBufferSize;
}
public Integer getSendBufferSize() {
return sendBufferSize;
}
public void setSendBufferSize(Integer sendBufferSize) {
this.sendBufferSize = sendBufferSize;
}
public Byte getConnectionCharsetNumber() {
return connectionCharsetNumber;
}
public void setConnectionCharsetNumber(Byte connectionCharsetNumber) {
this.connectionCharsetNumber = connectionCharsetNumber;
}
public String getConnectionCharset() {
return connectionCharset;
}
public void setConnectionCharset(String connectionCharset) {
this.connectionCharset = connectionCharset;
}
public IndexMode getIndexMode() {
return indexMode;
}
public void setIndexMode(IndexMode indexMode) {
this.indexMode = indexMode;
}
public String getDefaultDatabaseName() {
return defaultDatabaseName;
}
public void setDefaultDatabaseName(String defaultDatabaseName) {
this.defaultDatabaseName = defaultDatabaseName;
}
public Long getSlaveId() {
return slaveId;
}
public void setSlaveId(Long slaveId) {
this.slaveId = slaveId;
}
public Boolean getDetectingEnable() {
return detectingEnable;
}
public void setDetectingEnable(Boolean detectingEnable) {
this.detectingEnable = detectingEnable;
}
public String getDetectingSQL() {
return detectingSQL;
}
public void setDetectingSQL(String detectingSQL) {
this.detectingSQL = detectingSQL;
}
public Integer getDetectingIntervalInSeconds() {
return detectingIntervalInSeconds;
}
public void setDetectingIntervalInSeconds(Integer detectingIntervalInSeconds) {
this.detectingIntervalInSeconds = detectingIntervalInSeconds;
}
public Integer getDetectingTimeoutThresholdInSeconds() {
return detectingTimeoutThresholdInSeconds;
}
public void setDetectingTimeoutThresholdInSeconds(Integer detectingTimeoutThresholdInSeconds) {
this.detectingTimeoutThresholdInSeconds = detectingTimeoutThresholdInSeconds;
}
public Integer getDetectingRetryTimes() {
return detectingRetryTimes;
}
public void setDetectingRetryTimes(Integer detectingRetryTimes) {
this.detectingRetryTimes = detectingRetryTimes;
}
public StorageScavengeMode getStorageScavengeMode() {
return storageScavengeMode;
}
public void setStorageScavengeMode(StorageScavengeMode storageScavengeMode) {
this.storageScavengeMode = storageScavengeMode;
}
public String getScavengeSchdule() {
return scavengeSchdule;
}
public void setScavengeSchdule(String scavengeSchdule) {
this.scavengeSchdule = scavengeSchdule;
}
public String getApp() {
return app;
}
public String getGroup() {
return group;
}
public void setApp(String app) {
this.app = app;
}
public void setGroup(String group) {
this.group = group;
}
public String getMetaqStoreUri() {
return metaqStoreUri;
}
public void setMetaqStoreUri(String metaqStoreUri) {
this.metaqStoreUri = metaqStoreUri;
}
public Integer getTransactionSize() {
return transactionSize != null ? transactionSize : 1024;
}
public void setTransactionSize(Integer transactionSize) {
this.transactionSize = transactionSize;
}
public List<InetSocketAddress> getDbAddresses() {
if (dbAddresses == null) {
dbAddresses = new ArrayList<InetSocketAddress>();
if (masterAddress != null) {
dbAddresses.add(masterAddress);
}
if (standbyAddress != null) {
dbAddresses.add(standbyAddress);
}
}
return dbAddresses;
}
public List<List<DataSourcing>> getGroupDbAddresses() {
if (groupDbAddresses == null) {
groupDbAddresses = new ArrayList<List<DataSourcing>>();
if (dbAddresses != null) {
for (InetSocketAddress address : dbAddresses) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>();
groupAddresses.add(new DataSourcing(sourcingType, address));
groupDbAddresses.add(groupAddresses);
}
} else {
if (masterAddress != null) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>();
groupAddresses.add(new DataSourcing(sourcingType, masterAddress));
groupDbAddresses.add(groupAddresses);
}
if (standbyAddress != null) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>();
groupAddresses.add(new DataSourcing(sourcingType, standbyAddress));
groupDbAddresses.add(groupAddresses);
}
}
}
return groupDbAddresses;
}
public void setGroupDbAddresses(List<List<DataSourcing>> groupDbAddresses) {
this.groupDbAddresses = groupDbAddresses;
}
public void setDbAddresses(List<InetSocketAddress> dbAddresses) {
this.dbAddresses = dbAddresses;
}
public String getDbUsername() {
if (dbUsername == null) {
dbUsername = (masterUsername != null ? masterUsername : standbyUsername);
}
return dbUsername;
}
public void setDbUsername(String dbUsername) {
this.dbUsername = dbUsername;
}
public String getDbPassword() {
if (dbPassword == null) {
dbPassword = (masterPassword != null ? masterPassword : standbyPassword);
}
return dbPassword;
}
public void setDbPassword(String dbPassword) {
this.dbPassword = dbPassword;
}
public List<String> getPositions() {
if (positions == null) {
positions = new ArrayList<String>();
String masterPosition = buildPosition(masterLogfileName, masterLogfileOffest, masterTimestamp);
if (masterPosition != null) {
positions.add(masterPosition);
}
String standbyPosition = buildPosition(standbyLogfileName, standbyLogfileOffest, standbyTimestamp);
if (standbyPosition != null) {
positions.add(standbyPosition);
}
}
return positions;
}
public void setPositions(List<String> positions) {
this.positions = positions;
}
// ===========================兼容字段
private String buildPosition(String journalName, Long position, Long timestamp) {
StringBuilder masterBuilder = new StringBuilder();
if (StringUtils.isNotEmpty(journalName) || position != null || timestamp != null) {
masterBuilder.append('{');
if (StringUtils.isNotEmpty(journalName)) {
masterBuilder.append("\"journalName\":\"").append(journalName).append("\"");
}
if (position != null) {
if (masterBuilder.length() > 1) {
masterBuilder.append(",");
}
masterBuilder.append("\"position\":").append(position);
}
if (timestamp != null) {
if (masterBuilder.length() > 1) {
masterBuilder.append(",");
}
masterBuilder.append("\"timestamp\":").append(timestamp);
}
masterBuilder.append('}');
return masterBuilder.toString();
} else {
return null;
}
}
public void setMasterUsername(String masterUsername) {
this.masterUsername = masterUsername;
}
public void setMasterPassword(String masterPassword) {
this.masterPassword = masterPassword;
}
public void setStandbyAddress(InetSocketAddress standbyAddress) {
this.standbyAddress = standbyAddress;
}
public void setStandbyUsername(String standbyUsername) {
this.standbyUsername = standbyUsername;
}
public void setStandbyPassword(String standbyPassword) {
this.standbyPassword = standbyPassword;
}
public void setMasterLogfileName(String masterLogfileName) {
this.masterLogfileName = masterLogfileName;
}
public void setMasterLogfileOffest(Long masterLogfileOffest) {
this.masterLogfileOffest = masterLogfileOffest;
}
public void setMasterTimestamp(Long masterTimestamp) {
this.masterTimestamp = masterTimestamp;
}
public void setStandbyLogfileName(String standbyLogfileName) {
this.standbyLogfileName = standbyLogfileName;
}
public void setStandbyLogfileOffest(Long standbyLogfileOffest) {
this.standbyLogfileOffest = standbyLogfileOffest;
}
public void setStandbyTimestamp(Long standbyTimestamp) {
this.standbyTimestamp = standbyTimestamp;
}
public void setMasterAddress(InetSocketAddress masterAddress) {
this.masterAddress = masterAddress;
}
public Integer getFallbackIntervalInSeconds() {
return fallbackIntervalInSeconds == null ? 60 : fallbackIntervalInSeconds;
}
public void setFallbackIntervalInSeconds(Integer fallbackIntervalInSeconds) {
this.fallbackIntervalInSeconds = fallbackIntervalInSeconds;
}
public Boolean getHeartbeatHaEnable() {
return heartbeatHaEnable == null ? false : heartbeatHaEnable;
}
public void setHeartbeatHaEnable(Boolean heartbeatHaEnable) {
this.heartbeatHaEnable = heartbeatHaEnable;
}
public BatchMode getStorageBatchMode() {
return storageBatchMode == null ? BatchMode.MEMSIZE : storageBatchMode;
}
public void setStorageBatchMode(BatchMode storageBatchMode) {
this.storageBatchMode = storageBatchMode;
}
public Integer getMemoryStorageBufferMemUnit() {
return memoryStorageBufferMemUnit == null ? 1024 : memoryStorageBufferMemUnit;
}
public void setMemoryStorageBufferMemUnit(Integer memoryStorageBufferMemUnit) {
this.memoryStorageBufferMemUnit = memoryStorageBufferMemUnit;
}
public String getMediaGroup() {
return mediaGroup;
}
public void setMediaGroup(String mediaGroup) {
this.mediaGroup = mediaGroup;
}
public Long getZkClusterId() {
return zkClusterId;
}
public void setZkClusterId(Long zkClusterId) {
this.zkClusterId = zkClusterId;
}
public Boolean getDdlIsolation() {
return ddlIsolation == null ? false : ddlIsolation;
}
public void setDdlIsolation(Boolean ddlIsolation) {
this.ddlIsolation = ddlIsolation;
}
public Boolean getFilterTableError() {
return filterTableError == null ? false : filterTableError;
}
public void setFilterTableError(Boolean filterTableError) {
this.filterTableError = filterTableError;
}
public String getBlackFilter() {
return blackFilter;
}
public void setBlackFilter(String blackFilter) {
this.blackFilter = blackFilter;
}
public String toString() {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
}

22
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalStatus.java Normal file
View File

@@ -0,0 +1,22 @@
package com.alibaba.otter.canal.instance.manager.model;
/**
* 运行状态
*
* @author jianghang 2012-7-13 下午12:54:13
* @version 1.0.0
*/
public enum CanalStatus {
/** 启动 */
START,
/** 停止 */
STOP;
public boolean isStart() {
return this.equals(CanalStatus.START);
}
public boolean isStop() {
return this.equals(CanalStatus.STOP);
}
}

19
instance/pom.xml Normal file
View File

@@ -0,0 +1,19 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.19-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.instance</artifactId>
<packaging>pom</packaging>
<name>canal instance module for otter ${project.version}</name>
<modules>
<module>core</module>
<module>manager</module>
<module>spring</module>
</modules>
</project>

26
instance/spring/pom.xml Normal file
View File

@@ -0,0 +1,26 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.19-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.instance.spring</artifactId>
<packaging>jar</packaging>
<name>canal instance spring module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.instance.core</artifactId>
<version>${project.version}</version>
</dependency>
<!-- test dependency -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

181
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java Normal file
View File

@@ -0,0 +1,181 @@
package com.alibaba.otter.canal.instance.spring;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceSupport;
import com.alibaba.otter.canal.meta.CanalMetaManager;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.store.CanalEventStore;
import com.alibaba.otter.canal.store.model.Event;
/**
* 基于spring容器启动canal实例，方便独立于manager启动
*
* @author jianghang 2012-7-12 下午01:21:26
* @author zebin.xuzb
* @version 1.0.0
*/
public class CanalInstanceWithSpring extends CanalInstanceSupport implements CanalInstance {
private static final Logger logger = LoggerFactory.getLogger(CanalInstanceWithSpring.class);
private String destination;
private CanalEventParser eventParser;
private CanalEventSink<List<CanalEntry.Entry>> eventSink;
private CanalEventStore<Event> eventStore;
private CanalMetaManager metaManager;
private CanalAlarmHandler alarmHandler;
public String getDestination() {
return this.destination;
}
public CanalEventParser getEventParser() {
return this.eventParser;
}
public CanalEventSink<List<CanalEntry.Entry>> getEventSink() {
return this.eventSink;
}
public CanalEventStore<Event> getEventStore() {
return this.eventStore;
}
public CanalMetaManager getMetaManager() {
return this.metaManager;
}
public CanalAlarmHandler getAlarmHandler() {
return alarmHandler;
}
public boolean subscribeChange(ClientIdentity identity) {
if (StringUtils.isNotEmpty(identity.getFilter())) {
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());
boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
}
} else {
((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
}
}
// filter的处理规则
// a. parser处理数据过滤处理
// b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
// 后续内存版的一对多分发，可以考虑
return true;
}
protected void afterStartEventParser(CanalEventParser eventParser) {
super.afterStartEventParser(eventParser);
// 读取一下历史订阅的filter信息
List<ClientIdentity> clientIdentitys = metaManager.listAllSubscribeInfo(destination);
for (ClientIdentity clientIdentity : clientIdentitys) {
subscribeChange(clientIdentity);
}
}
public void start() {
super.start();
logger.info("start CannalInstance for {}-{} ", new Object[] { 1, destination });
if (!metaManager.isStart()) {
metaManager.start();
}
if (!eventStore.isStart()) {
eventStore.start();
}
if (!eventSink.isStart()) {
eventSink.start();
}
if (!eventParser.isStart()) {
beforeStartEventParser(eventParser);
eventParser.start();
afterStartEventParser(eventParser);
}
logger.info("start successful....");
}
public void stop() {
logger.info("stop CannalInstance for {}-{} ", new Object[] { 1, destination });
if (eventParser.isStart()) {
beforeStopEventParser(eventParser);
eventParser.stop();
afterStopEventParser(eventParser);
}
if (eventSink.isStart()) {
eventSink.stop();
}
if (eventStore.isStart()) {
eventStore.stop();
}
if (metaManager.isStart()) {
metaManager.stop();
}
if (alarmHandler.isStart()) {
alarmHandler.stop();
}
// if (zkClientx != null) {
// zkClientx.close();
// }
super.stop();
logger.info("stop successful....");
}
// ======== setter ========
public void setDestination(String destination) {
this.destination = destination;
}
public void setEventParser(CanalEventParser eventParser) {
this.eventParser = eventParser;
}
public void setEventSink(CanalEventSink<List<CanalEntry.Entry>> eventSink) {
this.eventSink = eventSink;
}
public void setEventStore(CanalEventStore<Event> eventStore) {
this.eventStore = eventStore;
}
public void setMetaManager(CanalMetaManager metaManager) {
this.metaManager = metaManager;
}
public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
this.alarmHandler = alarmHandler;
}
}

32
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java Normal file
View File

@@ -0,0 +1,32 @@
package com.alibaba.otter.canal.instance.spring;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.BeanFactoryAware;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
/**
* @author zebin.xuzb @ 2012-7-12
* @version 1.0.0
*/
public class SpringCanalInstanceGenerator implements CanalInstanceGenerator, BeanFactoryAware {
private String defaultName = "instance";
private BeanFactory beanFactory;
public CanalInstance generate(String destination) {
String beanName = destination;
if (!beanFactory.containsBean(beanName)) {
beanName = defaultName;
}
return (CanalInstance) beanFactory.getBean(beanName);
}
public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
this.beanFactory = beanFactory;
}
}

167
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java Normal file
View File

@@ -0,0 +1,167 @@
package com.alibaba.otter.canal.instance.spring.support;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.ResourceLoaderAware;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.util.Assert;
/**
* 扩展Spring的{@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer} ，增加默认值的功能。
* 例如：${placeholder:defaultValue}，假如placeholder的值不存在，则默认取得 defaultValue。
*
* @author jianghang 2013-1-24 下午03:37:56
* @version 1.0.0
*/
public class PropertyPlaceholderConfigurer extends org.springframework.beans.factory.config.PropertyPlaceholderConfigurer implements ResourceLoaderAware, InitializingBean {
private static final String PLACEHOLDER_PREFIX = "${";
private static final String PLACEHOLDER_SUFFIX = "}";
private ResourceLoader loader;
private String[] locationNames;
public PropertyPlaceholderConfigurer(){
setIgnoreUnresolvablePlaceholders(true);
}
public void setResourceLoader(ResourceLoader loader) {
this.loader = loader;
}
public void setLocationNames(String[] locations) {
this.locationNames = locations;
}
public void afterPropertiesSet() throws Exception {
Assert.notNull(loader, "no resourceLoader");
if (locationNames != null) {
for (int i = 0; i < locationNames.length; i++) {
locationNames[i] = resolveSystemPropertyPlaceholders(locationNames[i]);
}
}
if (locationNames != null) {
List<Resource> resources = new ArrayList<Resource>(locationNames.length);
for (String location : locationNames) {
location = trimToNull(location);
if (location != null) {
resources.add(loader.getResource(location));
}
}
super.setLocations(resources.toArray(new Resource[resources.size()]));
}
}
private String resolveSystemPropertyPlaceholders(String text) {
StringBuilder buf = new StringBuilder(text);
for (int startIndex = buf.indexOf(PLACEHOLDER_PREFIX); startIndex >= 0;) {
int endIndex = buf.indexOf(PLACEHOLDER_SUFFIX, startIndex + PLACEHOLDER_PREFIX.length());
if (endIndex != -1) {
String placeholder = buf.substring(startIndex + PLACEHOLDER_PREFIX.length(), endIndex);
int nextIndex = endIndex + PLACEHOLDER_SUFFIX.length();
try {
String value = resolveSystemPropertyPlaceholder(placeholder);
if (value != null) {
buf.replace(startIndex, endIndex + PLACEHOLDER_SUFFIX.length(), value);
nextIndex = startIndex + value.length();
} else {
System.err.println("Could not resolve placeholder '"
+ placeholder
+ "' in ["
+ text
+ "] as system property: neither system property nor environment variable found");
}
} catch (Throwable ex) {
System.err.println("Could not resolve placeholder '" + placeholder + "' in [" + text
+ "] as system property: " + ex);
}
startIndex = buf.indexOf(PLACEHOLDER_PREFIX, nextIndex);
} else {
startIndex = -1;
}
}
return buf.toString();
}
private String resolveSystemPropertyPlaceholder(String placeholder) {
DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
String value = System.getProperty(dp.placeholder);
if (value == null) {
value = System.getenv(dp.placeholder);
}
if (value == null) {
value = dp.defaultValue;
}
return value;
}
@Override
protected String resolvePlaceholder(String placeholder, Properties props, int systemPropertiesMode) {
DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
String value = super.resolvePlaceholder(dp.placeholder, props, systemPropertiesMode);
if (value == null) {
value = dp.defaultValue;
}
return trimToEmpty(value);
}
private static class DefaultablePlaceholder {
private final String defaultValue;
private final String placeholder;
public DefaultablePlaceholder(String placeholder){
int commaIndex = placeholder.indexOf(":");
String defaultValue = null;
if (commaIndex >= 0) {
defaultValue = trimToEmpty(placeholder.substring(commaIndex + 1));
placeholder = trimToEmpty(placeholder.substring(0, commaIndex));
}
this.placeholder = placeholder;
this.defaultValue = defaultValue;
}
}
private String trimToNull(String str) {
if (str == null) {
return null;
}
String result = str.trim();
if (result == null || result.length() == 0) {
return null;
}
return result;
}
public static String trimToEmpty(String str) {
if (str == null) {
return "";
}
return str.trim();
}
}

28
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java Normal file
View File

@@ -0,0 +1,28 @@
package com.alibaba.otter.canal.instance.spring.support;
import java.beans.PropertyEditorSupport;
import java.net.InetSocketAddress;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.PropertyEditorRegistrar;
import org.springframework.beans.PropertyEditorRegistry;
public class SocketAddressEditor extends PropertyEditorSupport implements PropertyEditorRegistrar {
public void registerCustomEditors(PropertyEditorRegistry registry) {
registry.registerCustomEditor(InetSocketAddress.class, this);
}
public void setAsText(String text) throws IllegalArgumentException {
String[] addresses = StringUtils.split(text, ":");
if (addresses.length > 0) {
if (addresses.length != 2) {
throw new RuntimeException("address[" + text + "] is illegal, eg.127.0.0.1:3306");
} else {
setValue(new InetSocketAddress(addresses[0], Integer.valueOf(addresses[1])));
}
} else {
setValue(null);
}
}
}

48
instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/DefaultSpringInstanceTest.java Normal file
View File

@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.instance.spring.integrated;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
/**
* @author zebin.xuzb @ 2012-7-13
* @version 1.0.0
*/
public class DefaultSpringInstanceTest {
private ApplicationContext context;
@Before
public void start() {
System.setProperty("canal.instance.destination", "retl");
context = new ClassPathXmlApplicationContext(new String[] { "spring/default-instance.xml" });
}
@After
public void close() {
if (context != null && context instanceof AbstractApplicationContext) {
((AbstractApplicationContext) context).close();
}
}
@Test
public void testInstance() {
CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
CanalInstance canalInstance = generator.generate("instance");
Assert.notNull(canalInstance);
canalInstance.start();
try {
Thread.sleep(10 * 1000);
} catch (InterruptedException e) {
}
canalInstance.stop();
}
}

48
instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/GroupSpringInstanceTest.java Normal file
View File

@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.instance.spring.integrated;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
/**
* @author zebin.xuzb @ 2012-7-13
* @version 1.0.0
*/
public class GroupSpringInstanceTest {
private ApplicationContext context;
@Before
public void start() {
System.setProperty("canal.instance.destination", "retl");
context = new ClassPathXmlApplicationContext(new String[] { "spring/group-instance.xml" });
}
@After
public void close() {
if (context != null && context instanceof AbstractApplicationContext) {
((AbstractApplicationContext) context).close();
}
}
@Test
public void testInstance() {
CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
CanalInstance canalInstance = generator.generate("instance");
Assert.notNull(canalInstance);
canalInstance.start();
try {
Thread.sleep(10 * 1000);
} catch (InterruptedException e) {
}
canalInstance.stop();
}
}

48
instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/MemorySpringInstanceTest.java Normal file
View File

@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.instance.spring.integrated;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
/**
* @author zebin.xuzb @ 2012-7-13
* @version 1.0.0
*/
public class MemorySpringInstanceTest {
private ApplicationContext context;
@Before
public void start() {
System.setProperty("canal.instance.destination", "retl");
context = new ClassPathXmlApplicationContext(new String[] { "spring/memory-instance.xml" });
}
@After
public void close() {
if (context != null && context instanceof AbstractApplicationContext) {
((AbstractApplicationContext) context).close();
}
}
@Test
public void testInstance() {
CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
CanalInstance canalInstance = generator.generate("instance");
Assert.notNull(canalInstance);
canalInstance.start();
try {
Thread.sleep(10 * 1000);
} catch (InterruptedException e) {
}
canalInstance.stop();
}
}

54
instance/spring/src/test/resources/canal.properties Normal file
View File

@@ -0,0 +1,54 @@
#################################################
######### common argument #############
#################################################
canal.id= 1
canal.ip=
canal.port= 11111
canal.zkServers=
# flush data to zk
canal.zookeeper.flush.period = 1000
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE
## detecing config
canal.instance.detecting.enable = false
canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false
# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size = 1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60
# network config
canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30
# binlog filter config
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
#################################################
######### destinations #############
#################################################
canal.destinations= example
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true
canal.auto.scan.interval = 5
# as far as possible to stop canal instance where client disconnect
canal.stopInstanceAsPossible = true
canal.instance.global.mode = spring
canal.instance.global.lazy = false
#canal.instance.global.manager.address = 127.0.0.1:1099
canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml

37
instance/spring/src/test/resources/retl/instance.properties Normal file
View File

@@ -0,0 +1,37 @@
canal.instance.detecting.enable = true
#################################################
## mysql serverId
canal.instance.mysql.slaveId = 1234
# position info
canal.instance.master.address = 127.0.0.1:3306
canal.instance.master.journal.name =
canal.instance.master.position =
canal.instance.master.timestamp =
canal.instance.master1.address = 127.0.0.1:3306
canal.instance.master1.journal.name =
canal.instance.master1.position =
canal.instance.master1.timestamp =
canal.instance.master2.address = 127.0.0.1:3306
canal.instance.master2.journal.name =
canal.instance.master2.position =
canal.instance.master2.timestamp =
#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
# username/password
canal.instance.dbUsername = xxxxx
canal.instance.dbPassword = xxxxx
canal.instance.defaultDatabaseName =
canal.instance.connectionCharset = UTF-8
# table regex
canal.instance.filter.regex = .*\\..*
#################################################

173
instance/spring/src/test/resources/spring/default-instance.xml Normal file
View File

@@ -0,0 +1,173 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref local="eventParser" />
</property>
<property name="eventSink">
<ref local="eventSink" />
</property>
<property name="eventStore">
<ref local="eventStore" />
</property>
<property name="metaManager">
<ref local="metaManager" />
</property>
<property name="alarmHandler">
<ref local="alarmHandler" />
</property>
</bean>
<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
<bean id="zkClientx" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean" >
<property name="targetClass" value="com.alibaba.otter.canal.common.zookeeper.ZkClientx" />
<property name="targetMethod" value="getZkClient" />
<property name="arguments">
<list>
<value>${canal.zkServers:127.0.0.1:2181}</value>
</list>
</property>
</bean>
<bean id="metaManager" class="com.alibaba.otter.canal.meta.PeriodMixedMetaManager">
<property name="zooKeeperMetaManager">
<bean class="com.alibaba.otter.canal.meta.ZooKeeperMetaManager">
<property name="zkClientx" ref="zkClientx" />
</bean>
</property>
<property name="period" value="${canal.zookeeper.flush.period:1000}" />
</bean>
<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
</bean>
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>
<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
</bean>
</property>
</bean>
</property>
<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />
<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
</bean>
</beans>

159
instance/spring/src/test/resources/spring/file-instance.xml Normal file
View File

@@ -0,0 +1,159 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref local="eventParser" />
</property>
<property name="eventSink">
<ref local="eventSink" />
</property>
<property name="eventStore">
<ref local="eventStore" />
</property>
<property name="metaManager">
<ref local="metaManager" />
</property>
<property name="alarmHandler">
<ref local="alarmHandler" />
</property>
</bean>
<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
<property name="dataDir" value="${canal.file.data.dir:../conf}" />
<property name="period" value="${canal.file.flush.period:1000}" />
</bean>
<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
</bean>
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>
<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
</bean>
</property>
</bean>
</property>
<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />
<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
</bean>
</beans>

236
instance/spring/src/test/resources/spring/group-instance.xml Normal file
View File

@@ -0,0 +1,236 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref local="eventParser" />
</property>
<property name="eventSink">
<ref local="eventSink" />
</property>
<property name="eventStore">
<ref local="eventStore" />
</property>
<property name="metaManager">
<ref local="metaManager" />
</property>
<property name="alarmHandler">
<ref local="alarmHandler" />
</property>
</bean>
<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
<bean id="metaManager" class="com.alibaba.otter.canal.meta.MemoryMetaManager" />
<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
</bean>
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.group.GroupEventParser">
<property name="eventParsers">
<list>
<ref bean="eventParser1" />
<ref bean="eventParser2" />
</list>
</property>
</bean>
<bean id="eventParser1" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>
<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />
<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.master1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.standby1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master1.journal.name}" />
<property name="position" value="${canal.instance.master1.position}" />
<property name="timestamp" value="${canal.instance.master1.timestamp}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby1.journal.name}" />
<property name="position" value="${canal.instance.standby1.position}" />
<property name="timestamp" value="${canal.instance.standby1.timestamp}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
</bean>
<bean id="eventParser2" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>
<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />
<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.master2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.standby2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master2.journal.name}" />
<property name="position" value="${canal.instance.master2.position}" />
<property name="timestamp" value="${canal.instance.master2.timestamp}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby2.journal.name}" />
<property name="position" value="${canal.instance.standby2.position}" />
<property name="timestamp" value="${canal.instance.standby2.timestamp}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
</bean>
</beans>

147
instance/spring/src/test/resources/spring/memory-instance.xml Normal file
View File

@@ -0,0 +1,147 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref local="eventParser" />
</property>
<property name="eventSink">
<ref local="eventSink" />
</property>
<property name="eventStore">
<ref local="eventStore" />
</property>
<property name="metaManager">
<ref local="metaManager" />
</property>
<property name="alarmHandler">
<ref local="alarmHandler" />
</property>
</bean>
<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
<bean id="metaManager" class="com.alibaba.otter.canal.meta.MemoryMetaManager" />
<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
</bean>
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
<property name="detectingIntervalInSeconds" value="${canal.instance.detecting.interval.time:5}" />
<property name="haController">
<bean class="com.alibaba.otter.canal.parse.ha.HeartBeatHAController">
<property name="detectingRetryTimes" value="${canal.instance.detecting.retry.threshold:3}" />
<property name="switchEnable" value="${canal.instance.detecting.heartbeatHaEnable:false}" />
</bean>
</property>
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>
<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
<!-- 网络链接参数 -->
<property name="receiveBufferSize" value="${canal.instance.network.receiveBufferSize:16384}" />
<property name="sendBufferSize" value="${canal.instance.network.sendBufferSize:16384}" />
<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<!-- failover切换时回退的时间 -->
<property name="fallbackIntervalInSeconds" value="${canal.instance.fallbackIntervalInSeconds:60}" />
<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<property name="standbyInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
</bean>
</property>
<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
</bean>
</property>
<property name="standbyPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.standby.journal.name}" />
<property name="position" value="${canal.instance.standby.position}" />
<property name="timestamp" value="${canal.instance.standby.timestamp}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
</bean>
</beans>