chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pyf2

Browse Source
This commit is contained in:
Payon
2018-08-07 19:29:27 +08:00
committed by GitHub
parent 02868d8fec 23f7b3b5d8
commit b1dc5bd59f
168 changed files with 19475 additions and 11726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
View File

@@ -6,7 +6,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.HistoryTableMetaCache;
import com.alibaba.otter.canal.meta.FileMixedMetaManager;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,13 +25,7 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.instance.core.AbstractCanalInstance;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*;
import com.alibaba.otter.canal.meta.MemoryMetaManager;
import com.alibaba.otter.canal.meta.PeriodMixedMetaManager;
import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
@@ -38,12 +36,7 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
import com.alibaba.otter.canal.parse.index.*;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
@@ -121,6 +114,11 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(getZkclientx());
((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
} else if (mode.isLocalFile()){
FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager();
fileMixedMetaManager.setDataDir(parameters.getDataDir());
fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod());
metaManager = fileMixedMetaManager;
} else {
throw new CanalException("unsupport MetaMode for " + mode);
}

26
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
View File

@@ -29,8 +29,10 @@ public class CanalParameter implements Serializable {
private Long zkClusterId; // zk集群id，为管理方便
private List<String> zkClusters; // zk集群地址
private String dataDir = "../conf"; // 默认本地文件数据的目录默认是conf
// meta相关参数
private MetaMode metaMode = MetaMode.MEMORY; // meta机制
private Integer metaFileFlushPeriod = 1000; // meta刷新间隔
// storage存储
private Integer transactionSize = 1024; // 支持处理的transaction事务大小
@@ -247,7 +249,9 @@ public class CanalParameter implements Serializable {
/** 文件存储模式 */
ZOOKEEPER,
/** 混合模式，内存+文件 */
MIXED;
MIXED,
/** 本地文件存储模式*/
LOCAL_FILE;
public boolean isMemory() {
return this.equals(MetaMode.MEMORY);
@@ -260,6 +264,10 @@ public class CanalParameter implements Serializable {
public boolean isMixed() {
return this.equals(MetaMode.MIXED);
}
public boolean isLocalFile(){
return this.equals(MetaMode.LOCAL_FILE);
}
}
public static enum IndexMode {
@@ -394,6 +402,22 @@ public class CanalParameter implements Serializable {
return storageMode;
}
public String getDataDir() {
return dataDir;
}
public void setDataDir(String dataDir) {
this.dataDir = dataDir;
}
public Integer getMetaFileFlushPeriod() {
return metaFileFlushPeriod;
}
public void setMetaFileFlushPeriod(Integer metaFileFlushPeriod) {
this.metaFileFlushPeriod = metaFileFlushPeriod;
}
public void setStorageMode(StorageMode storageMode) {
this.storageMode = storageMode;
}