repair issue #3401 (#3923)

* 解决OutAdapter单例导致的问题，如 1.多个es7 outAdapter的多线程问题（共享同一个es bulk request），2.多个es7 outAdapter共用一份es配置问题

* 1. 优化adapter和监听器之间的代码结构
2. 监听器只处理匹配的配置文件
3. bugfix-es adapter配置文件回调时漏写esVersion
理论上一个adapter实例对应一个config配置文件

Co-authored-by: agapple <jianghang.loujh@alibaba-inc.com>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -1,5 +1,11 @@
 package com.alibaba.otter.canal.adapter.launcher.loader;
 
+import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -8,7 +14,6 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
-
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -17,12 +22,6 @@ import org.springframework.core.env.Environment;
 import org.springframework.core.env.PropertySource;
 import org.springframework.core.env.StandardEnvironment;
 
-import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-
 /**
  * 外部适配器的加载器
  *
@@ -50,24 +49,34 @@ public class CanalAdapterLoader {
 
         for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
             for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+                int i = 0;
                 List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
                 List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+
                 for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                    // 保证一定有key
+                    if (config.getKey() == null) {
+                        String key = StringUtils.join(new String[]{Util.AUTO_GENERATED_PREFIX,
+                                        canalAdapter.getInstance(), group.getGroupId(), String.valueOf(i)},
+                                '-');
+                        config.setKey(key);
+                    }
+                    i++;
                     loadAdapter(config, canalOuterAdapters);
                 }
                 canalOuterAdapterGroups.add(canalOuterAdapters);
 
                 AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()
-                                                                                           + "|"
-                                                                                           + StringUtils.trimToEmpty(group.getGroupId()),
-                    f -> new AdapterProcessor(canalClientConfig,
-                        canalAdapter.getInstance(),
-                        group.getGroupId(),
-                        canalOuterAdapterGroups));
+                                + "|"
+                                + StringUtils.trimToEmpty(group.getGroupId()),
+                        f -> new AdapterProcessor(canalClientConfig,
+                                canalAdapter.getInstance(),
+                                group.getGroupId(),
+                                canalOuterAdapterGroups));
                 adapterProcessor.start();
 
                 logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
-                                                                                   + group.getGroupId());
+                        + group.getGroupId());
             }
         }
 
@@ -201,7 +210,7 @@ public class CanalAdapterLoader {
     private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
         try {
             OuterAdapter adapter;
-            adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
+            adapter = loader.getExtension(config.getName(), config.getKey());
 
             ClassLoader cl = Thread.currentThread().getContextClassLoader();
             // 替换ClassLoader

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java

@@ -1,15 +1,21 @@
 package com.alibaba.otter.canal.adapter.launcher.rest;
 
+import com.alibaba.otter.canal.adapter.launcher.common.EtlLock;
+import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
+import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
+import com.alibaba.otter.canal.client.adapter.support.FileName2KeyMapping;
+import com.alibaba.otter.canal.client.adapter.support.Result;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -19,14 +25,6 @@ import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.alibaba.otter.canal.adapter.launcher.common.EtlLock;
-import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
-import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
-import com.alibaba.otter.canal.client.adapter.support.Result;
-
 /**
  * 适配器操作Rest
  *
@@ -66,6 +64,9 @@ public class CommonRest {
     @PostMapping("/etl/{type}/{key}/{task}")
     public EtlResult etl(@PathVariable String type, @PathVariable String key, @PathVariable String task,
                          @RequestParam(name = "params", required = false) String params) {
+        if (key == null) {
+            key = FileName2KeyMapping.getKey(type, task);
+        }
         OuterAdapter adapter = loader.getExtension(type, key);
         String destination = adapter.getDestination(task);
         String lockKey = destination == null ? task : destination;
@@ -133,6 +134,9 @@ public class CommonRest {
      */
     @GetMapping("/count/{type}/{key}/{task}")
     public Map<String, Object> count(@PathVariable String type, @PathVariable String key, @PathVariable String task) {
+        if (key == null) {
+            key = FileName2KeyMapping.getKey(type, task);
+        }
         OuterAdapter adapter = loader.getExtension(type, key);
         return adapter.count(task);
     }

client-adapter/launcher/src/main/resources/bootstrap.yml

@@ -1,6 +1,6 @@
-#canal:
-#  manager:
-#    jdbc:
-#      url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
-#      username: root
-#      password: 121212
+canal:
+  manager:
+    jdbc:
+      url: jdbc:mysql://rm-bp1yjb2rh8249f1ft90130.mysql.rds.aliyuncs.com:3306/canal_manager?useUnicode=true
+      username: dev
+      password: dev123456