简化adapter配置

client-adapter/elasticsearch/pom.xml

@@ -71,6 +71,26 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/classes/es" overwrite="true" >
+                                    <fileset dir="${project.basedir}/target/classes/es" erroronmissingdir="true">
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java

@@ -63,12 +63,12 @@ public class ESAdapter implements OuterAdapter {
     @Override
     public void init(OuterAdapterConfig configuration) {
         try {
-            SPI spi = this.getClass().getAnnotation(SPI.class);
-            Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(spi.value());
+            Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load();
             // 过滤不匹配的key的配置
             esSyncConfigTmp.forEach((key, config) -> {
                 if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey())) {
+                    || (config.getOuterAdapterKey() != null
+                        && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
                     esSyncConfig.put(key, config);
                 }
             });

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java

@@ -1,25 +1,13 @@
 package com.alibaba.otter.canal.client.adapter.es.config;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 
 /**
  * ES 配置装载器
@@ -29,75 +17,25 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
  */
 public class ESSyncConfigLoader {
 
-    private static Logger                                   logger              = LoggerFactory
-        .getLogger(ESSyncConfigLoader.class);
+    private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class);
 
-    public static synchronized Map<String, ESSyncConfig> load(String name) {
+    public static synchronized Map<String, ESSyncConfig> load() {
         logger.info("## Start loading es mapping config ... ");
 
         Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>();
 
-        Collection<String> configs = AdapterConfigs.get("es");
-        if (configs == null) {
-            return esSyncConfig;
-        }
-        for (String c : configs) {
-            if (c == null) {
-                continue;
-            }
-            c = c.trim();
-            if (c.equals("") || c.startsWith("#")) {
-                continue;
-            }
-
-            ESSyncConfig config;
-            String configContent = null;
-
-            if (c.endsWith(".yml")) {
-                configContent = readConfigContent(name + "/" + c);
-            }
-
-            config = new Yaml().loadAs(configContent, ESSyncConfig.class);
-
+        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("es");
+        configContentMap.forEach((fileName, content) -> {
+            ESSyncConfig config = new Yaml().loadAs(content, ESSyncConfig.class);
             try {
                 config.validate();
             } catch (Exception e) {
-                throw new RuntimeException("ERROR Config: " + c, e);
+                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
             }
-            esSyncConfig.put(c, config);
-        }
+            esSyncConfig.put(fileName, config);
+        });
 
         logger.info("## ES mapping config loaded");
         return esSyncConfig;
     }
-
-    private static String readConfigContent(String config) {
-        InputStream in = null;
-        try {
-            // 先取本地文件，再取类路径
-            File configFile = new File("../conf/" + config);
-            if (configFile.exists()) {
-                in = new FileInputStream(configFile);
-            } else {
-                    in = ESSyncConfigLoader.class.getClassLoader().getResourceAsStream(config);
-            }
-            if (in == null) {
-                throw new RuntimeException("Config file: " + config + " not found.");
-            }
-
-            byte[] bytes = new byte[in.available()];
-            in.read(bytes);
-            return new String(bytes, StandardCharsets.UTF_8);
-        } catch (IOException e) {
-            throw new RuntimeException("Read es mapping config error ", e);
-        } finally {
-            try {
-                if (in != null) {
-                    in.close();
-                }
-            } catch (IOException e) {
-                // ignore
-            }
-        }
-    }
 }

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.es.test;
 
-import java.util.List;
 import java.util.Map;
 
 import org.junit.Assert;
@@ -9,21 +8,20 @@ import org.junit.Test;
 
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 
 public class ConfigLoadTest {
 
     @Before
     public void before() {
-        AdapterConfigs.put("es", "mytest_user.yml");
+        // AdapterConfigs.put("es", "mytest_user.yml");
         // 加载数据源连接池
         DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
     }
 
     @Test
     public void testLoad() {
-        Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load("es");
+        Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load();
         ESSyncConfig config = configMap.get("mytest_user.yml");
         Assert.assertNotNull(config);
         Assert.assertEquals("defaultDS", config.getDataSourceKey());

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java

@@ -2,28 +2,25 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
 
 import java.util.*;
 
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import javax.sql.DataSource;
+
 import org.elasticsearch.action.get.GetResponse;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
-import javax.sql.DataSource;
-
 public class LabelSyncJoinSub2Test {
 
     private ESAdapter esAdapter;
 
     @Before
     public void init() {
-        AdapterConfigs.put("es", "mytest_user_join_sub2.yml");
+        // AdapterConfigs.put("es", "mytest_user_join_sub2.yml");
         esAdapter = Common.init();
     }
 

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java

@@ -2,28 +2,25 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
 
 import java.util.*;
 
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import javax.sql.DataSource;
+
 import org.elasticsearch.action.get.GetResponse;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
-import javax.sql.DataSource;
-
 public class LabelSyncJoinSubTest {
 
     private ESAdapter esAdapter;
 
     @Before
     public void init() {
-        AdapterConfigs.put("es", "mytest_user_join_sub.yml");
+        // AdapterConfigs.put("es", "mytest_user_join_sub.yml");
         esAdapter = Common.init();
     }
 

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java

@@ -2,28 +2,25 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
 
 import java.util.*;
 
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import javax.sql.DataSource;
+
 import org.elasticsearch.action.get.GetResponse;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
-import javax.sql.DataSource;
-
 public class RoleSyncJoinOne2Test {
 
     private ESAdapter esAdapter;
 
     @Before
     public void init() {
-        AdapterConfigs.put("es", "mytest_user_join_one2.yml");
+        // AdapterConfigs.put("es", "mytest_user_join_one2.yml");
         esAdapter = Common.init();
     }
 

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java

@@ -4,14 +4,13 @@ import java.util.*;
 
 import javax.sql.DataSource;
 
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import org.elasticsearch.action.get.GetResponse;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
@@ -21,7 +20,7 @@ public class RoleSyncJoinOneTest {
 
     @Before
     public void init() {
-        AdapterConfigs.put("es", "mytest_user_join_one.yml");
+        // AdapterConfigs.put("es", "mytest_user_join_one.yml");
         esAdapter = Common.init();
     }
 

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java

@@ -4,16 +4,13 @@ import java.util.*;
 
 import javax.sql.DataSource;
 
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import org.elasticsearch.action.get.GetResponse;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
@@ -23,7 +20,7 @@ public class UserSyncJoinOneTest {
 
     @Before
     public void init() {
-        AdapterConfigs.put("es", "mytest_user_join_one.yml");
+//        AdapterConfigs.put("es", "mytest_user_join_one.yml");
         esAdapter = Common.init();
     }
 

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java

@@ -2,17 +2,13 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
 
 import java.util.*;
 
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import org.elasticsearch.action.get.GetResponse;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 
 public class UserSyncSingleTest {
@@ -21,7 +17,7 @@ public class UserSyncSingleTest {
 
     @Before
     public void init() {
-        AdapterConfigs.put("es", "mytest_user_single.yml");
+        // AdapterConfigs.put("es", "mytest_user_single.yml");
         esAdapter = Common.init();
     }
 

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml

@@ -1,10 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-esMapping:
-  _index: mytest_user
-  _type: _doc
-  _id: _id
-  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
-        b.role_name as _role_name, a.c_time as _c_time from user a
-        left join role b on b.id=a.role_id"
-  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one2.yml

@@ -1,10 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-esMapping:
-  _index: mytest_user
-  _type: _doc
-  _id: _id
-  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
-        concat(b.role_name,'_') as _role_name, a.c_time as _c_time from user a
-        left join role b on b.id=a.role_id"
-  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml

@@ -1,11 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-esMapping:
-  _index: mytest_user
-  _type: _doc
-  _id: _id
-  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
-        b.labels _labels, a.c_time as _c_time from user a
-        left join (select user_id, group_concat(label order by id desc separator ';') as labels from label
-        group by user_id) b on b.user_id=a.id"
-  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml

@@ -1,11 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-esMapping:
-  _index: mytest_user
-  _type: _doc
-  _id: _id
-  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
-        concat(b.labels, '_') as _labels, a.c_time as _c_time from user a
-        left join (select user_id, group_concat(label order by id desc separator ';') as labels from label
-        group by user_id) b on b.user_id=a.id"
-  commitBatch: 3000