chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed CanalAdmin instanceLog and polardbx columnar test

Browse Source
This commit is contained in:
jianghang.loujh
2025-03-19 15:51:27 +08:00
parent c998b3f100
commit e21e98aa83
9 changed files with 221 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java
View File

@@ -39,13 +39,10 @@ public class ES7xTemplate implements ESTemplate {
.getLogger(ESTemplate.class);
private static final int MAX_BATCH_SIZE = 1000;
private ESConnection esConnection;
private ESBulkRequest esBulkRequest;
// es 字段类型本地缓存
private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>();
private ESConnection esConnection;
private ESBulkRequest esBulkRequest;
public ES7xTemplate(ESConnection esConnection){
this.esConnection = esConnection;
@@ -65,8 +62,9 @@ public class ES7xTemplate implements ESTemplate {
if (mapping.getId() != null) {
String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) {
ESUpdateRequest updateRequest = esConnection.new ES7xUpdateRequest(mapping.getIndex(),
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
ESUpdateRequest updateRequest = esConnection.new ES7xUpdateRequest(mapping.getIndex(), pkVal.toString())
.setDoc(esFieldData)
.setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) {
updateRequest.setRouting(parentVal);
}
@@ -283,7 +281,8 @@ public class ES7xTemplate implements ESTemplate {
}
@Override
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> esFieldData) {
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
@@ -307,8 +306,8 @@ public class ES7xTemplate implements ESTemplate {
}
@Override
public Object getESDataFromDmlData(ESMapping mapping,String owner, Map<String, Object> dmlData, Map<String, Object> dmlOld,
Map<String, Object> esFieldData) {
public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
@@ -329,7 +328,7 @@ public class ES7xTemplate implements ESTemplate {
if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
}
}

153
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java
View File

@@ -55,16 +55,9 @@ import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
public class ESConnection {
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
public enum ESClientMode {
TRANSPORT, REST
}
private ESClientMode mode;
@SuppressWarnings("deprecation")
private TransportClient transportClient;
private RestHighLevelClient restHighLevelClient;
public ESConnection(String[] hosts, Map<String, String> properties, ESClientMode mode) throws UnknownHostException{
@@ -86,9 +79,10 @@ public class ESConnection {
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
String[] nameAndPwdArr = nameAndPwd.split(":");
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
restHighLevelClient = new RestHighLevelClient(restClientBuilder);
}
@@ -143,6 +137,78 @@ public class ESConnection {
return mappingMetaData;
}
// ------ get/set ------
public ESClientMode getMode() {
return mode;
}
public void setMode(ESClientMode mode) {
this.mode = mode;
}
public TransportClient getTransportClient() {
return transportClient;
}
public void setTransportClient(TransportClient transportClient) {
this.transportClient = transportClient;
}
public RestHighLevelClient getRestHighLevelClient() {
return restHighLevelClient;
}
public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
this.restHighLevelClient = restHighLevelClient;
}
private HttpHost createHttpHost(String uriStr) {
URI uri = URI.create(uriStr);
if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
return HttpHost.create(uri.toString());
}
try {
return HttpHost.create(new URI(uri
.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
.toString());
} catch (URISyntaxException ex) {
throw new IllegalStateException(ex);
}
}
public enum ESClientMode {
TRANSPORT, REST
}
public static class ES7xBulkResponse implements ESBulkRequest.ESBulkResponse {
private BulkResponse bulkResponse;
public ES7xBulkResponse(BulkResponse bulkResponse){
this.bulkResponse = bulkResponse;
}
@Override
public boolean hasFailures() {
return bulkResponse.hasFailures();
}
@Override
public void processFailBulkResponse(String errorMsg) {
for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
if (!itemResponse.isFailed()) {
continue;
}
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.error(itemResponse.getFailureMessage());
} else {
throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
}
}
}
}
public class ES7xIndexRequest implements ESBulkRequest.ESIndexRequest {
private IndexRequestBuilder indexRequestBuilder;
@@ -443,71 +509,4 @@ public class ESConnection {
this.bulkRequest = bulkRequest;
}
}
public static class ES7xBulkResponse implements ESBulkRequest.ESBulkResponse {
private BulkResponse bulkResponse;
public ES7xBulkResponse(BulkResponse bulkResponse){
this.bulkResponse = bulkResponse;
}
@Override
public boolean hasFailures() {
return bulkResponse.hasFailures();
}
@Override
public void processFailBulkResponse(String errorMsg) {
for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
if (!itemResponse.isFailed()) {
continue;
}
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.error(itemResponse.getFailureMessage());
} else {
throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
}
}
}
}
// ------ get/set ------
public ESClientMode getMode() {
return mode;
}
public void setMode(ESClientMode mode) {
this.mode = mode;
}
public TransportClient getTransportClient() {
return transportClient;
}
public void setTransportClient(TransportClient transportClient) {
this.transportClient = transportClient;
}
public RestHighLevelClient getRestHighLevelClient() {
return restHighLevelClient;
}
public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
this.restHighLevelClient = restHighLevelClient;
}
private HttpHost createHttpHost(String uriStr) {
URI uri = URI.create(uriStr);
if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
return HttpHost.create(uri.toString());
}
try {
return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(),
uri.getQuery(), uri.getFragment()).toString());
} catch (URISyntaxException ex) {
throw new IllegalStateException(ex);
}
}
}