chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed CanalAdmin instanceLog and polardbx columnar test

Browse Source
This commit is contained in:
jianghang.loujh
2025-03-19 15:51:27 +08:00
parent c998b3f100
commit e21e98aa83
9 changed files with 221 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/ES8xAdapter.java
View File

@@ -1,5 +1,14 @@
package com.alibaba.otter.canal.client.adapter.es8x;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.sql.DataSource;
import org.elasticsearch.action.search.SearchResponse;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es8x.etl.ESEtlService;
@@ -9,13 +18,6 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import org.elasticsearch.action.search.SearchResponse;
import javax.sql.DataSource;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* ES 8.x 外部适配器

7
client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ES8xTemplate.java
View File

@@ -39,13 +39,10 @@ public class ES8xTemplate implements ESTemplate {
.getLogger(ESTemplate.class);
private static final int MAX_BATCH_SIZE = 1000;
private ESConnection esConnection;
private ESBulkRequest esBulkRequest;
// es 字段类型本地缓存
private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>();
private ESConnection esConnection;
private ESBulkRequest esBulkRequest;
public ES8xTemplate(ESConnection esConnection){
this.esConnection = esConnection;

102
client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ESConnection.java
View File

@@ -147,6 +147,57 @@ public class ESConnection {
return mappingMetaData;
}
public RestHighLevelClient getRestHighLevelClient() {
return restHighLevelClient;
}
public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
this.restHighLevelClient = restHighLevelClient;
}
private HttpHost createHttpHost(String uriStr) {
URI uri = URI.create(uriStr);
if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
return HttpHost.create(uri.toString());
}
try {
return HttpHost.create(new URI(uri
.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
.toString());
} catch (URISyntaxException ex) {
throw new IllegalStateException(ex);
}
}
public static class ES8xBulkResponse implements ESBulkRequest.ESBulkResponse {
private BulkResponse bulkResponse;
public ES8xBulkResponse(BulkResponse bulkResponse){
this.bulkResponse = bulkResponse;
}
@Override
public boolean hasFailures() {
return bulkResponse.hasFailures();
}
@Override
public void processFailBulkResponse(String errorMsg) {
for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
if (!itemResponse.isFailed()) {
continue;
}
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.error(itemResponse.getFailureMessage());
} else {
throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
}
}
}
}
public class ES8xIndexRequest implements ESBulkRequest.ESIndexRequest {
private IndexRequestBuilder indexRequestBuilder;
@@ -395,55 +446,4 @@ public class ESConnection {
this.bulkRequest = bulkRequest;
}
}
public static class ES8xBulkResponse implements ESBulkRequest.ESBulkResponse {
private BulkResponse bulkResponse;
public ES8xBulkResponse(BulkResponse bulkResponse){
this.bulkResponse = bulkResponse;
}
@Override
public boolean hasFailures() {
return bulkResponse.hasFailures();
}
@Override
public void processFailBulkResponse(String errorMsg) {
for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
if (!itemResponse.isFailed()) {
continue;
}
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.error(itemResponse.getFailureMessage());
} else {
throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
}
}
}
}
public RestHighLevelClient getRestHighLevelClient() {
return restHighLevelClient;
}
public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
this.restHighLevelClient = restHighLevelClient;
}
private HttpHost createHttpHost(String uriStr) {
URI uri = URI.create(uriStr);
if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
return HttpHost.create(uri.toString());
}
try {
return HttpHost.create(new URI(uri
.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
.toString());
} catch (URISyntaxException ex) {
throw new IllegalStateException(ex);
}
}
}