rdb同步适配多数据源

2026-11-04 00:30:15 +08:00 · 2018-11-10 23:55:09 +08:00
parent 2d41bb9960
commit d7ab6e2dc1
17 changed files with 125 additions and 115 deletions
--- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
+++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
@@ -19,12 +19,13 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
 import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
 import com.alibaba.otter.canal.client.adapter.support.*;

-public abstract class RdbAdapter implements OuterAdapter {
+@SPI("rdb")
+public class RdbAdapter implements OuterAdapter {

-    private static Logger                       logger             = LoggerFactory.getLogger(RdbAdapter.class);
+    private static Logger                       logger     = LoggerFactory.getLogger(RdbAdapter.class);

-    private volatile Map<String, MappingConfig> rdbMapping         = null;                                     // 文件名对应配置
-    private volatile Map<String, MappingConfig> mappingConfigCache = null;                                     // 库名-表名对应配置
+    private volatile Map<String, MappingConfig> rdbMapping = new HashMap<>();                          // 文件名对应配置
+    private Map<String, MappingConfig>          mappingConfigCache;                                    // 库名-表名对应配置

    private DruidDataSource                     dataSource;

@@ -32,20 +33,20 @@ public abstract class RdbAdapter implements OuterAdapter {

    @Override
    public void init(OuterAdapterConfig configuration) {
-        if (mappingConfigCache == null) {
-            synchronized (MappingConfig.class) {
-                if (mappingConfigCache == null) {
-                    SPI spi = this.getClass().getAnnotation(SPI.class);
-                    rdbMapping = MappingConfigLoader.load(spi.value());
-                    mappingConfigCache = new HashMap<>();
-                    for (MappingConfig mappingConfig : rdbMapping.values()) {
-                        mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
-                                               + mappingConfig.getDbMapping().getDatabase() + "."
-                                               + mappingConfig.getDbMapping().getTable(),
-                            mappingConfig);
-                    }
-                }
+        SPI spi = this.getClass().getAnnotation(SPI.class);
+        Map<String, MappingConfig> rdbMappingTmp = MappingConfigLoader.load(spi.value());
+        // 过滤其他key的配置
+        rdbMappingTmp.forEach((key, mappingConfig) -> {
+            if (mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey())) {
+                rdbMapping.put(key, mappingConfig);
            }
+        });
+        mappingConfigCache = new HashMap<>();
+        for (MappingConfig mappingConfig : rdbMapping.values()) {
+            mappingConfigCache
+                .put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+                     + mappingConfig.getDbMapping().getDatabase() + "." + mappingConfig.getDbMapping().getTable(),
+                    mappingConfig);
        }

        Map<String, String> properties = configuration.getProperties();
--- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/MysqlAdapter.java
+++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/MysqlAdapter.java
@@ -1,8 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.rdb.adapters;
-
-import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-
-@SPI("mysql")
-public class MysqlAdapter extends RdbAdapter {
-}
--- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/OracleAdapter.java
+++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/OracleAdapter.java
@@ -1,8 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.rdb.adapters;
-
-import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-
-@SPI("oracle")
-public class OracleAdapter extends RdbAdapter {
-}
--- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/PostgresqlAdapter.java
+++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/PostgresqlAdapter.java
@@ -1,8 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.rdb.adapters;
-
-import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-
-@SPI("postgresql")
-public class PostgresqlAdapter extends RdbAdapter {
-}
--- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/SqlserverAdapter.java
+++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/adapters/SqlserverAdapter.java
@@ -1,8 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.rdb.adapters;
-
-import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-
-@SPI("sqlserver")
-public class SqlserverAdapter extends RdbAdapter {
-}
--- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java
+++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java
@@ -12,11 +12,13 @@ import java.util.Set;
 */
 public class MappingConfig {

-    private String    dataSourceKey; // 数据源key
+    private String    dataSourceKey;   // 数据源key

-    private String    destination;   // canal实例或MQ的topic
+    private String    destination;     // canal实例或MQ的topic

-    private DbMapping dbMapping;     // db映射配置
+    private String    outerAdapterKey; // 对应适配器的key
+
+    private DbMapping dbMapping;       // db映射配置

    public String getDataSourceKey() {
        return dataSourceKey;
@@ -26,6 +28,14 @@ public class MappingConfig {
        this.dataSourceKey = dataSourceKey;
    }

+    public String getOuterAdapterKey() {
+        return outerAdapterKey;
+    }
+
+    public void setOuterAdapterKey(String outerAdapterKey) {
+        this.outerAdapterKey = outerAdapterKey;
+    }
+
    public DbMapping getDbMapping() {
        return dbMapping;
    }
--- a/client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter
+++ b/client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter
@@ -1,4 +1 @@
-mysql=com.alibaba.otter.canal.client.adapter.rdb.adapters.MysqlAdapter
-postgresql=com.alibaba.otter.canal.client.adapter.rdb.adapters.PostgresqlAdapter
-oracle=com.alibaba.otter.canal.client.adapter.rdb.adapters.OracleAdapter
-sqlserver=com.alibaba.otter.canal.client.adapter.rdb.adapters.SqlserverAdapter
+rdb=com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter
--- a/client-adapter/rdb/src/main/resources/postgresql/mytest_user.yml
+++ b/client-adapter/rdb/src/main/resources/postgresql/mytest_user.yml
@@ -1,15 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-dbMapping:
-  database: mytest
-  table: user
-  targetTable: public.tb_user
-  targetPk:
-    id: id
-  mapAll: true
-#  targetColumns:
-#    id:
-#    name:
-#    role_id:
-#    c_time:
-#    test1:
--- a/client-adapter/rdb/src/main/resources/oracle/mytest_user.yml
+++ b/client-adapter/rdb/src/main/resources/oracle/mytest_user.yml
@@ -1,5 +1,6 @@
 dataSourceKey: defaultDS
 destination: example
+outerAdapterKey: oracle1
 dbMapping:
  database: mytest
  table: user
--- a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java
+++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java
@@ -1,7 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.rdb.test.sync;

 import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
-import com.alibaba.otter.canal.client.adapter.rdb.adapters.OracleAdapter;
 import com.alibaba.otter.canal.client.adapter.rdb.test.TestConstant;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
@@ -14,7 +13,8 @@ public class Common {
        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);

        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
-        outerAdapterConfig.setName("oracle");
+        outerAdapterConfig.setName("rdb");
+        outerAdapterConfig.setKey("oralce1");
        Map<String, String> properties = new HashMap<>();
        properties.put("jdbc.driveClassName", "oracle.jdbc.OracleDriver");
        properties.put("jdbc.url", "jdbc:oracle:thin:@127.0.0.1:49161:XE");
@@ -22,7 +22,7 @@ public class Common {
        properties.put("jdbc.password", "m121212");
        outerAdapterConfig.setProperties(properties);

-        RdbAdapter adapter = new OracleAdapter();
+        RdbAdapter adapter = new RdbAdapter();
        adapter.init(outerAdapterConfig);
        return adapter;
    }
--- a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/OracleSyncTest.java
+++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/OracleSyncTest.java
@@ -15,7 +15,7 @@ public class OracleSyncTest {

    @Before
    public void init() {
-        AdapterConfigs.put("oracle", "mytest_user.yml");
+        AdapterConfigs.put("rdb", "mytest_user.yml");
        rdbAdapter = Common.init();
    }