chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

format es8

Browse Source
This commit is contained in:
jianghang.loujh
2023-09-01 19:58:23 +08:00
parent 63dfbf65df
commit 2786e06ccf
3 changed files with 119 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/etl/ESEtlService.java
View File

@@ -1,5 +1,19 @@
package com.alibaba.otter.canal.client.adapter.es8x.etl; package com.alibaba.otter.canal.client.adapter.es8x.etl;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
@@ -15,18 +29,6 @@ import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig; import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.Util; import com.alibaba.otter.canal.client.adapter.support.Util;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import javax.sql.DataSource;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* ES ETL Service * ES ETL Service
@@ -40,7 +42,7 @@ public class ESEtlService extends AbstractEtlService {
private ESTemplate esTemplate; private ESTemplate esTemplate;
private ESSyncConfig config; private ESSyncConfig config;
public ESEtlService(ESConnection esConnection, ESSyncConfig config) { public ESEtlService(ESConnection esConnection, ESSyncConfig config){
super("ES", config); super("ES", config);
this.esConnection = esConnection; this.esConnection = esConnection;
this.esTemplate = new ES8xTemplate(esConnection); this.esTemplate = new ES8xTemplate(esConnection);

41
client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ES8xTemplate.java
View File

@@ -1,5 +1,22 @@
package com.alibaba.otter.canal.client.adapter.es8x.support; package com.alibaba.otter.canal.client.adapter.es8x.support;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.sql.DataSource;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
@@ -15,21 +32,6 @@ import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest; import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Util; import com.alibaba.otter.canal.client.adapter.support.Util;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.sql.DataSource;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class ES8xTemplate implements ESTemplate { public class ES8xTemplate implements ESTemplate {
@@ -45,7 +47,7 @@ public class ES8xTemplate implements ESTemplate {
// es 字段类型本地缓存 // es 字段类型本地缓存
private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>(); private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>();
public ES8xTemplate(ESConnection esConnection) { public ES8xTemplate(ESConnection esConnection){
this.esConnection = esConnection; this.esConnection = esConnection;
this.esBulkRequest = this.esConnection.new ES8xBulkRequest(); this.esBulkRequest = this.esConnection.new ES8xBulkRequest();
} }
@@ -281,7 +283,8 @@ public class ES8xTemplate implements ESTemplate {
} }
@Override @Override
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> esFieldData) { public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem(); SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
Object resultIdVal = null; Object resultIdVal = null;
@@ -305,8 +308,8 @@ public class ES8xTemplate implements ESTemplate {
} }
@Override @Override
public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData, Map<String, Object> dmlOld, public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
Map<String, Object> esFieldData) { Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem(); SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
Object resultIdVal = null; Object resultIdVal = null;

44
client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ESConnection.java
View File

@@ -1,6 +1,12 @@
package com.alibaba.otter.canal.client.adapter.es8x.support; package com.alibaba.otter.canal.client.adapter.es8x.support;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Map;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
@@ -30,12 +36,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Map;
/** /**
* ES 连接器, 只支持 Rest 方式 * ES 连接器, 只支持 Rest 方式
@@ -47,20 +48,22 @@ public class ESConnection {
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class); private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
private RestHighLevelClient restHighLevelClient; private RestHighLevelClient restHighLevelClient;
public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException { public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException{
HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new); HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
String nameAndPwd = properties.get("security.auth"); String nameAndPwd = properties.get("security.auth");
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) { if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
String[] nameAndPwdArr = nameAndPwd.split(":"); String[] nameAndPwdArr = nameAndPwd.split(":");
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1])); credentialsProvider.setCredentials(AuthScope.ANY,
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
} }
restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build()).setApiCompatibilityMode(true).build(); restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build()).setApiCompatibilityMode(true)
.build();
} }
public void close() { public void close() {
@@ -98,7 +101,7 @@ public class ESConnection {
private IndexRequest indexRequest; private IndexRequest indexRequest;
public ES8xIndexRequest(String index, String id) { public ES8xIndexRequest(String index, String id){
indexRequest = new IndexRequest(index); indexRequest = new IndexRequest(index);
indexRequest.id(id); indexRequest.id(id);
@@ -141,7 +144,7 @@ public class ESConnection {
private UpdateRequest updateRequest; private UpdateRequest updateRequest;
public ES8xUpdateRequest(String index, String id) { public ES8xUpdateRequest(String index, String id){
updateRequest = new UpdateRequest(index, id); updateRequest = new UpdateRequest(index, id);
} }
@@ -190,7 +193,7 @@ public class ESConnection {
private DeleteRequest deleteRequest; private DeleteRequest deleteRequest;
public ES8xDeleteRequest(String index, String id) { public ES8xDeleteRequest(String index, String id){
deleteRequest = new DeleteRequest(index, id); deleteRequest = new DeleteRequest(index, id);
@@ -221,7 +224,7 @@ public class ESConnection {
private SearchSourceBuilder sourceBuilder; private SearchSourceBuilder sourceBuilder;
public ESSearchRequest(String index) { public ESSearchRequest(String index){
searchRequest = new SearchRequest(index); searchRequest = new SearchRequest(index);
sourceBuilder = new SearchSourceBuilder(); sourceBuilder = new SearchSourceBuilder();
@@ -276,7 +279,7 @@ public class ESConnection {
private BulkRequest bulkRequest; private BulkRequest bulkRequest;
public ES8xBulkRequest() { public ES8xBulkRequest(){
bulkRequest = new BulkRequest(); bulkRequest = new BulkRequest();
@@ -347,7 +350,7 @@ public class ESConnection {
private BulkResponse bulkResponse; private BulkResponse bulkResponse;
public ES8xBulkResponse(BulkResponse bulkResponse) { public ES8xBulkResponse(BulkResponse bulkResponse){
this.bulkResponse = bulkResponse; this.bulkResponse = bulkResponse;
} }
@@ -372,7 +375,6 @@ public class ESConnection {
} }
} }
public RestHighLevelClient getRestHighLevelClient() { public RestHighLevelClient getRestHighLevelClient() {
return restHighLevelClient; return restHighLevelClient;
} }
@@ -387,7 +389,9 @@ public class ESConnection {
return HttpHost.create(uri.toString()); return HttpHost.create(uri.toString());
} }
try { try {
return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()).toString()); return HttpHost.create(new URI(uri
.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
.toString());
} catch (URISyntaxException ex) { } catch (URISyntaxException ex) {
throw new IllegalStateException(ex); throw new IllegalStateException(ex);
} }