新增es adapter rest client和认证机制 (#2046)

* 新增es adapter rest client和认证机制

* 修改getMapping位置

* 代码调整

* 代码调整

* 整理代码

* fix #2048

client-adapter/elasticsearch/pom.xml

@@ -26,12 +26,22 @@
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
-            <version>6.2.3</version>
+            <version>6.4.3</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>transport</artifactId>
-            <version>6.2.3</version>
+            <version>6.4.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-client</artifactId>
+            <version>6.4.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <version>6.4.3</version>
         </dependency>
 
         <dependency>

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.es;
 
-import java.net.InetAddress;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -13,10 +12,6 @@ import javax.sql.DataSource;
 
 import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
 
 import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
@@ -28,6 +23,7 @@ import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
 import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
 import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection;
 import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.*;
 
@@ -43,7 +39,7 @@ public class ESAdapter implements OuterAdapter {
     private Map<String, ESSyncConfig>              esSyncConfig        = new ConcurrentHashMap<>(); // 文件名对应配置
     private Map<String, Map<String, ESSyncConfig>> dbTableEsSyncConfig = new ConcurrentHashMap<>(); // schema-table对应配置
 
-    private TransportClient                        transportClient;
+    private ESConnection                           esConnection;
 
     private ESSyncService                          esSyncService;
 
@@ -51,10 +47,6 @@ public class ESAdapter implements OuterAdapter {
 
     private Properties                             envProperties;
 
-    public TransportClient getTransportClient() {
-        return transportClient;
-    }
-
     public ESSyncService getEsSyncService() {
         return esSyncService;
     }
@@ -67,6 +59,14 @@ public class ESAdapter implements OuterAdapter {
         return dbTableEsSyncConfig;
     }
 
+    public ESConnection getEsConnection() {
+        return esConnection;
+    }
+
+    public void setEsConnection(ESConnection esConnection) {
+        this.esConnection = esConnection;
+    }
+
     @Override
     public void init(OuterAdapterConfig configuration, Properties envProperties) {
         try {
@@ -119,17 +119,16 @@ public class ESAdapter implements OuterAdapter {
             }
 
             Map<String, String> properties = configuration.getProperties();
-            Settings.Builder settingBuilder = Settings.builder();
+
-            properties.forEach(settingBuilder::put);
-            Settings settings = settingBuilder.build();
-            transportClient = new PreBuiltTransportClient(settings);
             String[] hostArray = configuration.getHosts().split(",");
-            for (String host : hostArray) {
+            String mode = properties.get("mode");
-                int i = host.indexOf(":");
+            if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
-                transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
+                esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
-                    Integer.parseInt(host.substring(i + 1))));
+            } else {
+                esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
             }
-            ESTemplate esTemplate = new ESTemplate(transportClient);
+
+            ESTemplate esTemplate = new ESTemplate(esConnection);
             esSyncService = new ESSyncService(esTemplate);
 
             esConfigMonitor = new ESConfigMonitor();
@@ -177,7 +176,7 @@ public class ESAdapter implements OuterAdapter {
         ESSyncConfig config = esSyncConfig.get(task);
         if (config != null) {
             DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-            ESEtlService esEtlService = new ESEtlService(transportClient, config);
+            ESEtlService esEtlService = new ESEtlService(esConnection, config);
             if (dataSource != null) {
                 return esEtlService.importData(params);
             } else {
@@ -191,7 +190,7 @@ public class ESAdapter implements OuterAdapter {
             for (ESSyncConfig configTmp : esSyncConfig.values()) {
                 // 取所有的destination为task的配置
                 if (configTmp.getDestination().equals(task)) {
-                    ESEtlService esEtlService = new ESEtlService(transportClient, configTmp);
+                    ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
                     EtlResult etlRes = esEtlService.importData(params);
                     if (!etlRes.getSucceeded()) {
                         resSuccess = false;
@@ -220,10 +219,8 @@ public class ESAdapter implements OuterAdapter {
     public Map<String, Object> count(String task) {
         ESSyncConfig config = esSyncConfig.get(task);
         ESMapping mapping = config.getEsMapping();
-        SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).size(0)
-            .setTypes(mapping.get_type())
+            .getResponse();
-            .setSize(0)
-            .get();
 
         long rowCount = response.getHits().getTotalHits();
         Map<String, Object> res = new LinkedHashMap<>();
@@ -237,8 +234,8 @@ public class ESAdapter implements OuterAdapter {
         if (esConfigMonitor != null) {
             esConfigMonitor.destroy();
         }
-        if (transportClient != null) {
+        if (esConnection != null) {
-            transportClient.close();
+            esConnection.close();
         }
     }
 

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java

@@ -11,12 +11,8 @@ import javax.sql.DataSource;
 
 import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateRequestBuilder;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -24,6 +20,11 @@ import org.elasticsearch.search.SearchHit;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESBulkRequest;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESIndexRequest;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESSearchRequest;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESUpdateRequest;
 import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
 import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
@@ -38,14 +39,14 @@ import com.alibaba.otter.canal.client.adapter.support.Util;
  */
 public class ESEtlService extends AbstractEtlService {
 
-    private TransportClient transportClient;
+    private ESConnection esConnection;
     private ESTemplate   esTemplate;
     private ESSyncConfig config;
 
-    public ESEtlService(TransportClient transportClient, ESSyncConfig config){
+    public ESEtlService(ESConnection esConnection, ESSyncConfig config){
         super("ES", config);
-        this.transportClient = transportClient;
+        this.esConnection = esConnection;
-        this.esTemplate = new ESTemplate(transportClient);
+        this.esTemplate = new ESTemplate(esConnection);
         this.config = config;
     }
 
@@ -79,7 +80,7 @@ public class ESEtlService extends AbstractEtlService {
             Util.sqlRS(ds, sql, values, rs -> {
                 int count = 0;
                 try {
-                    BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
+                    ESBulkRequest esBulkRequest = this.esConnection.new ESBulkRequest();
 
                     long batchBegin = System.currentTimeMillis();
                     while (rs.next()) {
@@ -132,41 +133,41 @@ public class ESEtlService extends AbstractEtlService {
                         if (idVal != null) {
                             String parentVal = (String) esFieldData.remove("$parent_routing");
                             if (mapping.isUpsert()) {
-                                UpdateRequestBuilder updateRequestBuilder = transportClient
+                                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(
-                                    .prepareUpdate(mapping.get_index(), mapping.get_type(), idVal.toString())
+                                    mapping.get_index(),
-                                    .setDoc(esFieldData)
+                                    mapping.get_type(),
-                                    .setDocAsUpsert(true);
+                                    idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+
                                 if (StringUtils.isNotEmpty(parentVal)) {
-                                    updateRequestBuilder.setRouting(parentVal);
+                                    esUpdateRequest.setRouting(parentVal);
                                 }
-                                bulkRequestBuilder.add(updateRequestBuilder);
+
+                                esBulkRequest.add(esUpdateRequest);
                             } else {
-                                IndexRequestBuilder indexRequestBuilder = transportClient
+                                ESIndexRequest esIndexRequest = this.esConnection.new ESIndexRequest(mapping
-                                    .prepareIndex(mapping.get_index(), mapping.get_type(), idVal.toString())
+                                    .get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
-                                    .setSource(esFieldData);
                                 if (StringUtils.isNotEmpty(parentVal)) {
-                                    indexRequestBuilder.setRouting(parentVal);
+                                    esIndexRequest.setRouting(parentVal);
                                 }
-                                bulkRequestBuilder.add(indexRequestBuilder);
+                                esBulkRequest.add(esIndexRequest);
                             }
                         } else {
                             idVal = esFieldData.get(mapping.getPk());
-                            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                                .setTypes(mapping.get_type())
+                                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
-                                .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
+                                    .size(10000);
-                                .setSize(10000)
+                            SearchResponse response = esSearchRequest.getResponse();
-                                .get();
                             for (SearchHit hit : response.getHits()) {
-                                bulkRequestBuilder.add(
+                                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping
-                                    transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId())
+                                    .get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
-                                        .setDoc(esFieldData));
+                                esBulkRequest.add(esUpdateRequest);
                             }
                         }
 
-                        if (bulkRequestBuilder.numberOfActions() % mapping.getCommitBatch() == 0
+                        if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
-                            && bulkRequestBuilder.numberOfActions() > 0) {
+                            && esBulkRequest.numberOfActions() > 0) {
                             long esBatchBegin = System.currentTimeMillis();
-                            BulkResponse rp = bulkRequestBuilder.execute().actionGet();
+                            BulkResponse rp = esBulkRequest.bulk();
                             if (rp.hasFailures()) {
                                 this.processFailBulkResponse(rp);
                             }
@@ -175,19 +176,19 @@ public class ESEtlService extends AbstractEtlService {
                                 logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
                                     (System.currentTimeMillis() - batchBegin),
                                     (System.currentTimeMillis() - esBatchBegin),
-                                    bulkRequestBuilder.numberOfActions(),
+                                    esBulkRequest.numberOfActions(),
                                     mapping.get_index());
                             }
                             batchBegin = System.currentTimeMillis();
-                            bulkRequestBuilder = transportClient.prepareBulk();
+                            esBulkRequest.resetBulk();
                         }
                         count++;
                         impCount.incrementAndGet();
                     }
 
-                    if (bulkRequestBuilder.numberOfActions() > 0) {
+                    if (esBulkRequest.numberOfActions() > 0) {
                         long esBatchBegin = System.currentTimeMillis();
-                        BulkResponse rp = bulkRequestBuilder.execute().actionGet();
+                        BulkResponse rp = esBulkRequest.bulk();
                         if (rp.hasFailures()) {
                             this.processFailBulkResponse(rp);
                         }
@@ -195,7 +196,7 @@ public class ESEtlService extends AbstractEtlService {
                             logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
                                 (System.currentTimeMillis() - batchBegin),
                                 (System.currentTimeMillis() - esBatchBegin),
-                                bulkRequestBuilder.numberOfActions(),
+                                esBulkRequest.numberOfActions(),
                                 mapping.get_index());
                         }
                     }

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESConnection.java

@@ -0,0 +1,468 @@
+package com.alibaba.otter.canal.client.adapter.es.support;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteRequestBuilder;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateRequestBuilder;
+import org.elasticsearch.client.*;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * ES 连接器, Transport Rest 两种方式
+ *
+ * @author rewerma 2019-08-01
+ * @version 1.0.0
+ */
+public class ESConnection {
+
+    private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
+
+    public enum ESClientMode {
+                              TRANSPORT, REST
+    }
+
+    private ESClientMode        mode;
+
+    private TransportClient     transportClient;
+
+    private RestHighLevelClient restHighLevelClient;
+
+    public ESConnection(String[] hosts, Map<String, String> properties, ESClientMode mode) throws UnknownHostException{
+        this.mode = mode;
+        if (mode == ESClientMode.TRANSPORT) {
+            Settings.Builder settingBuilder = Settings.builder();
+            properties.forEach(settingBuilder::put);
+            Settings settings = settingBuilder.build();
+            transportClient = new PreBuiltTransportClient(settings);
+            for (String host : hosts) {
+                int i = host.indexOf(":");
+                transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
+                    Integer.parseInt(host.substring(i + 1))));
+            }
+        } else {
+            HttpHost[] httpHosts = new HttpHost[hosts.length];
+            for (int i = 0; i < hosts.length; i++) {
+                String host = hosts[i];
+                int j = host.indexOf(":");
+                HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
+                    Integer.parseInt(host.substring(j + 1)));
+                httpHosts[i] = httpHost;
+            }
+            RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
+            String nameAndPwd = properties.get("security.auth");
+            if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
+                String[] nameAndPwdArr = nameAndPwd.split(":");
+                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+                credentialsProvider.setCredentials(AuthScope.ANY,
+                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
+                restClientBuilder.setHttpClientConfigCallback(
+                    httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+            }
+            restHighLevelClient = new RestHighLevelClient(restClientBuilder);
+        }
+    }
+
+    public void close() {
+        if (mode == ESClientMode.TRANSPORT) {
+            transportClient.close();
+        } else {
+            try {
+                restHighLevelClient.close();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    public MappingMetaData getMapping(String index, String type) {
+        MappingMetaData mappingMetaData = null;
+        if (mode == ESClientMode.TRANSPORT) {
+            ImmutableOpenMap<String, MappingMetaData> mappings;
+            try {
+                mappings = transportClient.admin()
+                    .cluster()
+                    .prepareState()
+                    .execute()
+                    .actionGet()
+                    .getState()
+                    .getMetaData()
+                    .getIndices()
+                    .get(index)
+                    .getMappings();
+            } catch (NullPointerException e) {
+                throw new IllegalArgumentException("Not found the mapping info of index: " + index);
+            }
+            mappingMetaData = mappings.get(type);
+
+        } else {
+            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings;
+            try {
+                GetMappingsRequest request = new GetMappingsRequest();
+                request.indices(index);
+                GetMappingsResponse response;
+                // try {
+                // response = restHighLevelClient
+                // .indices()
+                // .getMapping(request, RequestOptions.DEFAULT);
+                // // 6.4以下版本直接使用该接口会报错
+                // } catch (Exception e) {
+                // logger.warn("Low ElasticSearch version for getMapping");
+                response = RestHighLevelClientExt.getMapping(restHighLevelClient, request, RequestOptions.DEFAULT);
+                // }
+
+                mappings = response.mappings();
+            } catch (NullPointerException e) {
+                throw new IllegalArgumentException("Not found the mapping info of index: " + index);
+            } catch (IOException e) {
+                logger.error(e.getMessage(), e);
+                return null;
+            }
+            mappingMetaData = mappings.get(index).get(type);
+        }
+        return mappingMetaData;
+    }
+
+    public class ESIndexRequest {
+
+        private IndexRequestBuilder indexRequestBuilder;
+
+        private IndexRequest        indexRequest;
+
+        public ESIndexRequest(String index, String type, String id){
+            if (mode == ESClientMode.TRANSPORT) {
+                indexRequestBuilder = transportClient.prepareIndex(index, type, id);
+            } else {
+                indexRequest = new IndexRequest(index, type, id);
+            }
+        }
+
+        public ESIndexRequest setSource(Map<String, ?> source) {
+            if (mode == ESClientMode.TRANSPORT) {
+                indexRequestBuilder.setSource(source);
+            } else {
+                indexRequest.source(source);
+            }
+            return this;
+        }
+
+        public ESIndexRequest setRouting(String routing) {
+            if (mode == ESClientMode.TRANSPORT) {
+                indexRequestBuilder.setRouting(routing);
+            } else {
+                indexRequest.routing(routing);
+            }
+            return this;
+        }
+
+        public IndexRequestBuilder getIndexRequestBuilder() {
+            return indexRequestBuilder;
+        }
+
+        public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) {
+            this.indexRequestBuilder = indexRequestBuilder;
+        }
+
+        public IndexRequest getIndexRequest() {
+            return indexRequest;
+        }
+
+        public void setIndexRequest(IndexRequest indexRequest) {
+            this.indexRequest = indexRequest;
+        }
+    }
+
+    public class ESUpdateRequest {
+
+        private UpdateRequestBuilder updateRequestBuilder;
+
+        private UpdateRequest        updateRequest;
+
+        public ESUpdateRequest(String index, String type, String id){
+            if (mode == ESClientMode.TRANSPORT) {
+                updateRequestBuilder = transportClient.prepareUpdate(index, type, id);
+            } else {
+                updateRequest = new UpdateRequest(index, type, id);
+            }
+        }
+
+        public ESUpdateRequest setDoc(Map source) {
+            if (mode == ESClientMode.TRANSPORT) {
+                updateRequestBuilder.setDoc(source);
+            } else {
+                updateRequest.doc(source);
+            }
+            return this;
+        }
+
+        public ESUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) {
+            if (mode == ESClientMode.TRANSPORT) {
+                updateRequestBuilder.setDocAsUpsert(shouldUpsertDoc);
+            } else {
+                updateRequest.docAsUpsert(shouldUpsertDoc);
+            }
+            return this;
+        }
+
+        public ESUpdateRequest setRouting(String routing) {
+            if (mode == ESClientMode.TRANSPORT) {
+                updateRequestBuilder.setRouting(routing);
+            } else {
+                updateRequest.routing(routing);
+            }
+            return this;
+        }
+
+        public UpdateRequestBuilder getUpdateRequestBuilder() {
+            return updateRequestBuilder;
+        }
+
+        public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) {
+            this.updateRequestBuilder = updateRequestBuilder;
+        }
+
+        public UpdateRequest getUpdateRequest() {
+            return updateRequest;
+        }
+
+        public void setUpdateRequest(UpdateRequest updateRequest) {
+            this.updateRequest = updateRequest;
+        }
+    }
+
+    public class ESDeleteRequest {
+
+        private DeleteRequestBuilder deleteRequestBuilder;
+
+        private DeleteRequest        deleteRequest;
+
+        public ESDeleteRequest(String index, String type, String id){
+            if (mode == ESClientMode.TRANSPORT) {
+                deleteRequestBuilder = transportClient.prepareDelete(index, type, id);
+            } else {
+                deleteRequest = new DeleteRequest(index, type, id);
+            }
+        }
+
+        public DeleteRequestBuilder getDeleteRequestBuilder() {
+            return deleteRequestBuilder;
+        }
+
+        public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) {
+            this.deleteRequestBuilder = deleteRequestBuilder;
+        }
+
+        public DeleteRequest getDeleteRequest() {
+            return deleteRequest;
+        }
+
+        public void setDeleteRequest(DeleteRequest deleteRequest) {
+            this.deleteRequest = deleteRequest;
+        }
+    }
+
+    public class ESSearchRequest {
+
+        private SearchRequestBuilder searchRequestBuilder;
+
+        private SearchRequest        searchRequest;
+
+        private SearchSourceBuilder  sourceBuilder;
+
+        public ESSearchRequest(String index, String... types){
+            if (mode == ESClientMode.TRANSPORT) {
+                searchRequestBuilder = transportClient.prepareSearch(index).setTypes(types);
+            } else {
+                searchRequest = new SearchRequest(index).types(types);
+                sourceBuilder = new SearchSourceBuilder();
+            }
+        }
+
+        public ESSearchRequest setQuery(QueryBuilder queryBuilder) {
+            if (mode == ESClientMode.TRANSPORT) {
+                searchRequestBuilder.setQuery(queryBuilder);
+            } else {
+                sourceBuilder.query(queryBuilder);
+            }
+            return this;
+        }
+
+        public ESSearchRequest size(int size) {
+            if (mode == ESClientMode.TRANSPORT) {
+                searchRequestBuilder.setSize(size);
+            } else {
+                sourceBuilder.size(size);
+            }
+            return this;
+        }
+
+        public SearchResponse getResponse() {
+            if (mode == ESClientMode.TRANSPORT) {
+                return searchRequestBuilder.get();
+            } else {
+                searchRequest.source(sourceBuilder);
+                try {
+                    return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
+        public SearchRequestBuilder getSearchRequestBuilder() {
+            return searchRequestBuilder;
+        }
+
+        public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) {
+            this.searchRequestBuilder = searchRequestBuilder;
+        }
+
+        public SearchRequest getSearchRequest() {
+            return searchRequest;
+        }
+
+        public void setSearchRequest(SearchRequest searchRequest) {
+            this.searchRequest = searchRequest;
+        }
+    }
+
+    public class ESBulkRequest {
+
+        private BulkRequestBuilder bulkRequestBuilder;
+
+        private BulkRequest        bulkRequest;
+
+        public ESBulkRequest(){
+            if (mode == ESClientMode.TRANSPORT) {
+                bulkRequestBuilder = transportClient.prepareBulk();
+            } else {
+                bulkRequest = new BulkRequest();
+            }
+        }
+
+        public void resetBulk() {
+            if (mode == ESClientMode.TRANSPORT) {
+                bulkRequestBuilder = transportClient.prepareBulk();
+            } else {
+                bulkRequest = new BulkRequest();
+            }
+        }
+
+        public ESBulkRequest add(ESIndexRequest esIndexRequest) {
+            if (mode == ESClientMode.TRANSPORT) {
+                bulkRequestBuilder.add(esIndexRequest.indexRequestBuilder);
+            } else {
+                bulkRequest.add(esIndexRequest.indexRequest);
+            }
+            return this;
+        }
+
+        public ESBulkRequest add(ESUpdateRequest esUpdateRequest) {
+            if (mode == ESClientMode.TRANSPORT) {
+                bulkRequestBuilder.add(esUpdateRequest.updateRequestBuilder);
+            } else {
+                bulkRequest.add(esUpdateRequest.updateRequest);
+            }
+            return this;
+        }
+
+        public ESBulkRequest add(ESDeleteRequest esDeleteRequest) {
+            if (mode == ESClientMode.TRANSPORT) {
+                bulkRequestBuilder.add(esDeleteRequest.deleteRequestBuilder);
+            } else {
+                bulkRequest.add(esDeleteRequest.deleteRequest);
+            }
+            return this;
+        }
+
+        public int numberOfActions() {
+            if (mode == ESClientMode.TRANSPORT) {
+                return bulkRequestBuilder.numberOfActions();
+            } else {
+                return bulkRequest.numberOfActions();
+            }
+        }
+
+        public BulkResponse bulk() {
+            if (mode == ESClientMode.TRANSPORT) {
+                return bulkRequestBuilder.execute().actionGet();
+            } else {
+                try {
+                    return restHighLevelClient.bulk(bulkRequest);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
+        public BulkRequestBuilder getBulkRequestBuilder() {
+            return bulkRequestBuilder;
+        }
+
+        public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) {
+            this.bulkRequestBuilder = bulkRequestBuilder;
+        }
+
+        public BulkRequest getBulkRequest() {
+            return bulkRequest;
+        }
+
+        public void setBulkRequest(BulkRequest bulkRequest) {
+            this.bulkRequest = bulkRequest;
+        }
+    }
+
+    // ------ get/set ------
+    public ESClientMode getMode() {
+        return mode;
+    }
+
+    public void setMode(ESClientMode mode) {
+        this.mode = mode;
+    }
+
+    public TransportClient getTransportClient() {
+        return transportClient;
+    }
+
+    public void setTransportClient(TransportClient transportClient) {
+        this.transportClient = transportClient;
+    }
+
+    public RestHighLevelClient getRestHighLevelClient() {
+        return restHighLevelClient;
+    }
+
+    public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java

@@ -10,14 +10,9 @@ import javax.sql.DataSource;
 
 import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateRequestBuilder;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
@@ -30,6 +25,7 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.*;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Util;
 
@@ -45,21 +41,21 @@ public class ESTemplate {
 
     private static final int MAX_BATCH_SIZE = 1000;
 
-    private TransportClient     transportClient;
+    private ESConnection esConnection;
 
-    private BulkRequestBuilder  bulkRequestBuilder;
+    private ESBulkRequest esBulkRequest;
 
-    public ESTemplate(TransportClient transportClient){
+    public ESTemplate(ESConnection esConnection) {
-        this.transportClient = transportClient;
+        this.esConnection = esConnection;
-        this.bulkRequestBuilder = transportClient.prepareBulk();
+        this.esBulkRequest = this.esConnection.new ESBulkRequest();
     }
 
-    public BulkRequestBuilder getBulk() {
+    public ESBulkRequest getBulk() {
-        return bulkRequestBuilder;
+        return esBulkRequest;
     }
 
     public void resetBulkRequestBuilder() {
-        this.bulkRequestBuilder = this.transportClient.prepareBulk();
+        this.esBulkRequest.resetBulk();
     }
 
     /**
@@ -73,33 +69,33 @@ public class ESTemplate {
         if (mapping.get_id() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                UpdateRequestBuilder updateRequestBuilder = transportClient
+                ESUpdateRequest updateRequest = esConnection.new ESUpdateRequest(mapping.get_index(),
-                    .prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString())
+                        mapping.get_type(),
-                    .setDoc(esFieldData)
+                        pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-                    .setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
-                    updateRequestBuilder.setRouting(parentVal);
+                    updateRequest.setRouting(parentVal);
                 }
-                getBulk().add(updateRequestBuilder);
+                getBulk().add(updateRequest);
             } else {
-                IndexRequestBuilder indexRequestBuilder = transportClient
+                ESIndexRequest indexRequest = esConnection.new ESIndexRequest(mapping.get_index(),
-                    .prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString())
+                        mapping.get_type(),
-                    .setSource(esFieldData);
+                        pkVal.toString()).setSource(esFieldData);
                 if (StringUtils.isNotEmpty(parentVal)) {
-                    indexRequestBuilder.setRouting(parentVal);
+                    indexRequest.setRouting(parentVal);
                 }
-                getBulk().add(indexRequestBuilder);
+                getBulk().add(indexRequest);
             }
             commitBulk();
         } else {
-            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                .setTypes(mapping.get_type())
+                    mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
-                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+            SearchResponse response = esSearchRequest.getResponse();
-                .setSize(10000)
+
-                .get();
             for (SearchHit hit : response.getHits()) {
-                getBulk().add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId())
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    .setDoc(esFieldData));
+                        mapping.get_type(),
+                        hit.getId()).setDoc(esFieldData);
+                getBulk().add(esUpdateRequest);
                 commitBulk();
             }
         }
@@ -143,7 +139,7 @@ public class ESTemplate {
             sql.append("_v.").append(fieldName).append("=? AND ");
             values.add(value);
         });
-        //TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
+        // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
         int len = sql.length();
         sql.delete(len - 4, len);
         Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
@@ -174,17 +170,20 @@ public class ESTemplate {
      */
     public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
         if (mapping.get_id() != null) {
-            getBulk().add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString()));
+            ESDeleteRequest esDeleteRequest = this.esConnection.new ESDeleteRequest(mapping.get_index(),
+                    mapping.get_type(),
+                    pkVal.toString());
+            getBulk().add(esDeleteRequest);
             commitBulk();
         } else {
-            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                .setTypes(mapping.get_type())
+                    mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
-                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+            SearchResponse response = esSearchRequest.getResponse();
-                .setSize(10000)
-                .get();
             for (SearchHit hit : response.getHits()) {
-                getBulk().add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId())
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    .setDoc(esFieldData));
+                        mapping.get_type(),
+                        hit.getId()).setDoc(esFieldData);
+                getBulk().add(esUpdateRequest);
                 commitBulk();
             }
         }
@@ -196,7 +195,7 @@ public class ESTemplate {
      */
     public void commit() {
         if (getBulk().numberOfActions() > 0) {
-            BulkResponse response = getBulk().execute().actionGet();
+            BulkResponse response = getBulk().bulk();
             if (response.hasFailures()) {
                 for (BulkItemResponse itemResponse : response.getItems()) {
                     if (!itemResponse.isFailed()) {
@@ -227,32 +226,31 @@ public class ESTemplate {
         if (mapping.get_id() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                UpdateRequestBuilder updateRequestBuilder = transportClient
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    .prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString())
+                        mapping.get_type(),
-                    .setDoc(esFieldData)
+                        pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-                    .setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
-                    updateRequestBuilder.setRouting(parentVal);
+                    esUpdateRequest.setRouting(parentVal);
                 }
-                getBulk().add(updateRequestBuilder);
+                getBulk().add(esUpdateRequest);
             } else {
-                UpdateRequestBuilder updateRequestBuilder = transportClient
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    .prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString())
+                        mapping.get_type(),
-                    .setDoc(esFieldData);
+                        pkVal.toString()).setDoc(esFieldData);
                 if (StringUtils.isNotEmpty(parentVal)) {
-                    updateRequestBuilder.setRouting(parentVal);
+                    esUpdateRequest.setRouting(parentVal);
                 }
-                getBulk().add(updateRequestBuilder);
+                getBulk().add(esUpdateRequest);
             }
         } else {
-            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                .setTypes(mapping.get_type())
+                    mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
-                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+            SearchResponse response = esSearchRequest.getResponse();
-                .setSize(10000)
-                .get();
             for (SearchHit hit : response.getHits()) {
-                getBulk().add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId())
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    .setDoc(esFieldData));
+                        mapping.get_type(),
+                        hit.getId()).setDoc(esFieldData);
+                getBulk().add(esUpdateRequest);
             }
         }
     }
@@ -490,23 +488,11 @@ public class ESTemplate {
     private String getEsType(ESMapping mapping, String fieldName) {
         String key = mapping.get_index() + "-" + mapping.get_type();
         Map<String, String> fieldType = esFieldTypes.get(key);
-        if (fieldType == null) {
+        if (fieldType != null) {
-            ImmutableOpenMap<String, MappingMetaData> mappings;
+            return fieldType.get(fieldName);
-            try {
+        } else {
-                mappings = transportClient.admin()
+            MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index(), mapping.get_type());
-                    .cluster()
+
-                    .prepareState()
-                    .execute()
-                    .actionGet()
-                    .getState()
-                    .getMetaData()
-                    .getIndices()
-                    .get(mapping.get_index())
-                    .getMappings();
-            } catch (NullPointerException e) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
-            }
-            MappingMetaData mappingMetaData = mappings.get(mapping.get_type());
             if (mappingMetaData == null) {
                 throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
             }
@@ -524,8 +510,9 @@ public class ESTemplate {
                 }
             }
             esFieldTypes.put(key, fieldType);
-        }
 
             return fieldType.get(fieldName);
         }
+    }
+
 }

client-adapter/elasticsearch/src/main/java/org/elasticsearch/client/RequestConvertersExt.java

@@ -0,0 +1,30 @@
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.common.Strings;
+
+import java.io.IOException;
+
+/**
+ * RequestConverters扩展
+ *
+ * @author rewerma 2019-08-01
+ * @version 1.0.0
+ */
+public class RequestConvertersExt {
+
+    /**
+     * 修改 getMappings 去掉request参数
+     *
+     * @param getMappingsRequest
+     * @return
+     * @throws IOException
+     */
+    static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOException {
+        String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices();
+        String[] types = getMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.types();
+
+        return new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping", types));
+    }
+}

client-adapter/elasticsearch/src/main/java/org/elasticsearch/client/RestHighLevelClientExt.java

@@ -0,0 +1,29 @@
+package org.elasticsearch.client;
+
+import static java.util.Collections.emptySet;
+
+import java.io.IOException;
+
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+
+/**
+ * RestHighLevelClient扩展
+ *
+ * @author rewerma 2019-08-01
+ * @version 1.0.0
+ */
+public class RestHighLevelClientExt {
+
+    public static GetMappingsResponse getMapping(RestHighLevelClient restHighLevelClient,
+                                                 GetMappingsRequest getMappingsRequest,
+                                                 RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest,
+            RequestConvertersExt::getMappings,
+            options,
+            GetMappingsResponse::fromXContent,
+            emptySet());
+
+    }
+
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java

@@ -18,6 +18,7 @@ import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
+
 @Ignore
 public class LabelSyncJoinSub2Test {
 
@@ -59,7 +60,10 @@ public class LabelSyncJoinSub2Test {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("b;a_", response.getSource().get("_labels"));
     }
 
@@ -97,7 +101,10 @@ public class LabelSyncJoinSub2Test {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
     }
 
@@ -129,7 +136,10 @@ public class LabelSyncJoinSub2Test {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("b_", response.getSource().get("_labels"));
     }
 }

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java

@@ -18,6 +18,7 @@ import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
+
 @Ignore
 public class LabelSyncJoinSubTest {
 
@@ -59,7 +60,10 @@ public class LabelSyncJoinSubTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("b;a", response.getSource().get("_labels"));
     }
 
@@ -97,7 +101,10 @@ public class LabelSyncJoinSubTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("b;aa", response.getSource().get("_labels"));
     }
 
@@ -129,7 +136,10 @@ public class LabelSyncJoinSubTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("b", response.getSource().get("_labels"));
     }
 }

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java

@@ -18,6 +18,7 @@ import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
+
 @Ignore
 public class RoleSyncJoinOne2Test {
 
@@ -57,7 +58,10 @@ public class RoleSyncJoinOne2Test {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("admin_", response.getSource().get("_role_name"));
     }
 
@@ -94,7 +98,10 @@ public class RoleSyncJoinOne2Test {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("admin3_", response.getSource().get("_role_name"));
     }
 }

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java

@@ -58,7 +58,7 @@ public class RoleSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
         Assert.assertEquals("admin", response.getSource().get("_role_name"));
     }
 
@@ -95,7 +95,7 @@ public class RoleSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
         Assert.assertEquals("admin2", response.getSource().get("_role_name"));
     }
 
@@ -133,7 +133,7 @@ public class RoleSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
         Assert.assertEquals("operator", response.getSource().get("_role_name"));
 
         Common.sqlExe(ds, "update user set role_id=1 where id=1");
@@ -158,7 +158,7 @@ public class RoleSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2);
 
-        GetResponse response2 = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response2 = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
         Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
     }
 
@@ -190,7 +190,8 @@ public class RoleSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
         Assert.assertNull(response.getSource().get("_role_name"));
     }
 }

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java

@@ -18,6 +18,7 @@ import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
+
 @Ignore
 public class UserSyncJoinOneTest {
 
@@ -59,7 +60,10 @@ public class UserSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("Eric_", response.getSource().get("_name"));
     }
 
@@ -95,7 +99,10 @@ public class UserSyncJoinOneTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("Eric2_", response.getSource().get("_name"));
     }
 }

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java

@@ -15,6 +15,7 @@ import org.junit.Test;
 import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
+
 @Ignore
 public class UserSyncSingleTest {
 
@@ -52,7 +53,10 @@ public class UserSyncSingleTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("Eric", response.getSource().get("_name"));
     }
 
@@ -85,7 +89,10 @@ public class UserSyncSingleTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertEquals("Eric2", response.getSource().get("_name"));
     }
 
@@ -115,7 +122,10 @@ public class UserSyncSingleTest {
 
         esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
 
-        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        GetResponse response = esAdapter.getEsConnection()
+            .getTransportClient()
+            .prepareGet("mytest_user", "_doc", "1")
+            .get();
         Assert.assertNull(response.getSource());
     }
 

client-adapter/launcher/src/main/resources/application.yml

@@ -7,10 +7,10 @@ spring:
     default-property-inclusion: non_null
 
 canal.conf:
-  mode: kafka # kafka rocketMQ
+  mode: tcp # kafka rocketMQ
-#  canalServerHost: 127.0.0.1:11111
+  canalServerHost: 127.0.0.1:11111
 #  zookeeperHosts: slave1:2181
-  mqServers: 127.0.0.1:9092 #or rocketmq
+#  mqServers: 127.0.0.1:9092 #or rocketmq
 #  flatMessage: true
   batchSize: 500
   syncBatchSize: 1000
@@ -58,6 +58,9 @@ canal.conf:
 #          hbase.zookeeper.property.clientPort: 2181
 #          zookeeper.znode.parent: /hbase
 #      - name: es
-#        hosts: 127.0.0.1:9300
+#        hosts: 127.0.0.1:9300 # 127.0.0.1:9200 for rest mode
 #        properties:
+#          mode: transport # or rest
+#          # security.auth: test:123456 #  only used for rest mode
 #          cluster.name: elasticsearch
+

server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java

@@ -177,6 +177,7 @@ public class CanalKafkaProducer implements CanalMQProducer {
 
                     // 每条记录需要flush
                     produce(topicName, records, true);
+                    records.clear();
                 }
             }
         }