support otter use tableMetaTSDB

2026-11-04 00:30:15 +08:00 · 2018-08-08 18:54:57 +08:00
parent 377a72e1c0
commit 9e816bc48f
21 changed files with 149 additions and 611 deletions
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
@@ -6,11 +6,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;

-
-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.HistoryTableMetaCache;
-
-import com.alibaba.otter.canal.meta.FileMixedMetaManager;
-
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,7 +20,14 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
 import com.alibaba.otter.canal.instance.core.AbstractCanalInstance;
 import com.alibaba.otter.canal.instance.manager.model.Canal;
 import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
-import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode;
+import com.alibaba.otter.canal.meta.FileMixedMetaManager;
 import com.alibaba.otter.canal.meta.MemoryMetaManager;
 import com.alibaba.otter.canal.meta.PeriodMixedMetaManager;
 import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
@@ -36,7 +38,12 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
 import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
 import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
 import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-import com.alibaba.otter.canal.parse.index.*;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
+import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
+import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
 import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
 import com.alibaba.otter.canal.sink.entry.EntryEventSink;
@@ -114,7 +121,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
            ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
            zooKeeperMetaManager.setZkClientx(getZkclientx());
            ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
-        } else if (mode.isLocalFile()){
+        } else if (mode.isLocalFile()) {
            FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager();
            fileMixedMetaManager.setDataDir(parameters.getDataDir());
            fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod());
@@ -241,13 +248,6 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
            mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
            // 数据库信息参数
            mysqlEventParser.setSlaveId(parameters.getSlaveId());
-            mysqlEventParser.setTableMetaStorageFactory(parameters.getTableMetaStorageFactory());
-            // Ctrip callback
-//            mysqlEventParser.setCallback(parameters.getCallback());
-//            HistoryTableMetaCache cache = new HistoryTableMetaCache();
-//            cache.init(parameters.getEntries());
-//            mysqlEventParser.setHistoryTableMetaCache(cache);
-
            if (!CollectionUtils.isEmpty(dbAddresses)) {
                mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
                    parameters.getDbUsername(),
--- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
@@ -5,7 +5,6 @@ import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.List;

-import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.builder.ToStringBuilder;

@@ -94,6 +93,10 @@ public class CanalParameter implements Serializable {
    private Boolean                  filterTableError                   = Boolean.FALSE;             // 是否忽略表解析异常
    private String                   blackFilter                        = null;                      // 匹配黑名单,忽略解析

+    private Boolean                  tsdbEnable                         = Boolean.FALSE;             // 是否开启tableMetaTSDB
+    private String                   tsdbJdbcUrl;
+    private String                   tsdbJdbcUserName;
+    private String                   tsdbJdbcPassword;
    // ================================== 兼容字段处理
    private InetSocketAddress        masterAddress;                                                  // 主库信息
    private String                   masterUsername;                                                 // 帐号
@@ -109,9 +112,6 @@ public class CanalParameter implements Serializable {
    private Long                     standbyLogfileOffest               = null;
    private Long                     standbyTimestamp                   = null;

-    // Ctrip Table Meta
-    TableMetaStorageFactory tableMetaStorageFactory;
-
    public static enum RunMode {

        /** 嵌入式 */
@@ -250,7 +250,7 @@ public class CanalParameter implements Serializable {
        ZOOKEEPER,
        /** 混合模式，内存+文件 */
        MIXED,
-        /** 本地文件存储模式*/
+        /** 本地文件存储模式 */
        LOCAL_FILE;

        public boolean isMemory() {
@@ -265,7 +265,7 @@ public class CanalParameter implements Serializable {
            return this.equals(MetaMode.MIXED);
        }

-        public boolean isLocalFile(){
+        public boolean isLocalFile() {
            return this.equals(MetaMode.LOCAL_FILE);
        }
    }
@@ -887,12 +887,36 @@ public class CanalParameter implements Serializable {
        this.blackFilter = blackFilter;
    }

-    public TableMetaStorageFactory getTableMetaStorageFactory() {
-        return tableMetaStorageFactory;
+    public Boolean getTsdbEnable() {
+        return tsdbEnable;
    }

-    public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) {
-        this.tableMetaStorageFactory = tableMetaStorageFactory;
+    public void setTsdbEnable(Boolean tsdbEnable) {
+        this.tsdbEnable = tsdbEnable;
+    }
+
+    public String getTsdbJdbcUrl() {
+        return tsdbJdbcUrl;
+    }
+
+    public void setTsdbJdbcUrl(String tsdbJdbcUrl) {
+        this.tsdbJdbcUrl = tsdbJdbcUrl;
+    }
+
+    public String getTsdbJdbcUserName() {
+        return tsdbJdbcUserName;
+    }
+
+    public void setTsdbJdbcUserName(String tsdbJdbcUserName) {
+        this.tsdbJdbcUserName = tsdbJdbcUserName;
+    }
+
+    public String getTsdbJdbcPassword() {
+        return tsdbJdbcPassword;
+    }
+
+    public void setTsdbJdbcPassword(String tsdbJdbcPassword) {
+        this.tsdbJdbcPassword = tsdbJdbcPassword;
    }

    public String toString() {