mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
format es8
This commit is contained in:
@@ -1,5 +1,19 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.es8x.etl;
|
package com.alibaba.otter.canal.client.adapter.es8x.etl;
|
||||||
|
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
|
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
|
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
|
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
|
||||||
@@ -15,18 +29,6 @@ import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
|
|||||||
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.elasticsearch.search.SearchHit;
|
|
||||||
|
|
||||||
import javax.sql.DataSource;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ES ETL Service
|
* ES ETL Service
|
||||||
|
|||||||
@@ -1,5 +1,22 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.es8x.support;
|
package com.alibaba.otter.canal.client.adapter.es8x.support;
|
||||||
|
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.ConcurrentMap;
|
||||||
|
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.cluster.metadata.MappingMetadata;
|
||||||
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
|
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
|
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
|
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
|
||||||
@@ -15,21 +32,6 @@ import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
|
|||||||
import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
|
import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
|
||||||
import org.elasticsearch.cluster.metadata.MappingMetadata;
|
|
||||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.elasticsearch.search.SearchHit;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import javax.sql.DataSource;
|
|
||||||
import java.sql.ResultSet;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.ConcurrentMap;
|
|
||||||
|
|
||||||
public class ES8xTemplate implements ESTemplate {
|
public class ES8xTemplate implements ESTemplate {
|
||||||
|
|
||||||
@@ -281,7 +283,8 @@ public class ES8xTemplate implements ESTemplate {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> esFieldData) {
|
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
|
||||||
|
Map<String, Object> esFieldData) {
|
||||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||||
Object resultIdVal = null;
|
Object resultIdVal = null;
|
||||||
@@ -305,8 +308,8 @@ public class ES8xTemplate implements ESTemplate {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData, Map<String, Object> dmlOld,
|
public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
|
||||||
Map<String, Object> esFieldData) {
|
Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
|
||||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||||
Object resultIdVal = null;
|
Object resultIdVal = null;
|
||||||
|
|||||||
@@ -1,6 +1,12 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.es8x.support;
|
package com.alibaba.otter.canal.client.adapter.es8x.support;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.net.UnknownHostException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.http.HttpHost;
|
import org.apache.http.HttpHost;
|
||||||
import org.apache.http.auth.AuthScope;
|
import org.apache.http.auth.AuthScope;
|
||||||
@@ -30,12 +36,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.IOException;
|
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
|
||||||
import java.net.URI;
|
|
||||||
import java.net.URISyntaxException;
|
|
||||||
import java.net.UnknownHostException;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ES 连接器, 只支持 Rest 方式
|
* ES 连接器, 只支持 Rest 方式
|
||||||
@@ -47,7 +48,6 @@ public class ESConnection {
|
|||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
|
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
|
||||||
|
|
||||||
|
|
||||||
private RestHighLevelClient restHighLevelClient;
|
private RestHighLevelClient restHighLevelClient;
|
||||||
|
|
||||||
public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException{
|
public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException{
|
||||||
@@ -57,10 +57,13 @@ public class ESConnection {
|
|||||||
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
|
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
|
||||||
String[] nameAndPwdArr = nameAndPwd.split(":");
|
String[] nameAndPwdArr = nameAndPwd.split(":");
|
||||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
|
credentialsProvider.setCredentials(AuthScope.ANY,
|
||||||
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
|
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
|
||||||
|
restClientBuilder.setHttpClientConfigCallback(
|
||||||
|
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
|
||||||
}
|
}
|
||||||
restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build()).setApiCompatibilityMode(true).build();
|
restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build()).setApiCompatibilityMode(true)
|
||||||
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() {
|
public void close() {
|
||||||
@@ -372,7 +375,6 @@ public class ESConnection {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public RestHighLevelClient getRestHighLevelClient() {
|
public RestHighLevelClient getRestHighLevelClient() {
|
||||||
return restHighLevelClient;
|
return restHighLevelClient;
|
||||||
}
|
}
|
||||||
@@ -387,7 +389,9 @@ public class ESConnection {
|
|||||||
return HttpHost.create(uri.toString());
|
return HttpHost.create(uri.toString());
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()).toString());
|
return HttpHost.create(new URI(uri
|
||||||
|
.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
|
||||||
|
.toString());
|
||||||
} catch (URISyntaxException ex) {
|
} catch (URISyntaxException ex) {
|
||||||
throw new IllegalStateException(ex);
|
throw new IllegalStateException(ex);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user