chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix Configuration property name '-id' is not valid on canal 1.1.7 alpha 3 #4879 (#4887)

Browse Source 
* Fix Configuration property name '-id' is not valid

* Fix Configuration property name '-id' is not valid
读取原配置文件
This commit is contained in:
cooper lee
2023-09-21 10:53:47 +08:00
committed by GitHub
parent f9f810226d
commit cda330e49e
15 changed files with 198 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java
View File

@@ -59,12 +59,12 @@ public class ES6xAdapter extends ESAdapter {
public Map<String, Object> count(String task) {
ESSyncConfig config = esSyncConfig.get(task);
ESSyncConfig.ESMapping mapping = config.getEsMapping();
SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).size(0)
SearchResponse response = this.esConnection.new ESSearchRequest(mapping.getIndex(), mapping.getType()).size(0)
.getResponse();
long rowCount = response.getHits().getTotalHits();
Map<String, Object> res = new LinkedHashMap<>();
res.put("esIndex", mapping.get_index());
res.put("esIndex", mapping.getIndex());
res.put("count", rowCount);
return res;
}

26
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java
View File

@@ -51,7 +51,7 @@ public class ESEtlService extends AbstractEtlService {
public EtlResult importData(List<String> params) {
ESMapping mapping = config.getEsMapping();
logger.info("start etl to import data to index: {}", mapping.get_index());
logger.info("start etl to import data to index: {}", mapping.getIndex());
String sql = mapping.getSql();
return importData(sql, params);
}
@@ -78,7 +78,7 @@ public class ESEtlService extends AbstractEtlService {
}
// 如果是主键字段则不插入
if (fieldItem.getFieldName().equals(mapping.get_id())) {
if (fieldItem.getFieldName().equals(mapping.getId())) {
idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
} else {
Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
@@ -117,8 +117,8 @@ public class ESEtlService extends AbstractEtlService {
if (idVal != null) {
String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) {
@@ -127,8 +127,8 @@ public class ESEtlService extends AbstractEtlService {
esBulkRequest.add(esUpdateRequest);
} else {
ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(),
mapping.get_type(),
ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.getIndex(),
mapping.getType(),
idVal.toString()).setSource(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) {
esIndexRequest.setRouting(parentVal);
@@ -137,12 +137,12 @@ public class ESEtlService extends AbstractEtlService {
}
} else {
idVal = esFieldData.get(mapping.getPk());
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
hit.getId()).setDoc(esFieldData);
esBulkRequest.add(esUpdateRequest);
}
@@ -160,7 +160,7 @@ public class ESEtlService extends AbstractEtlService {
(System.currentTimeMillis() - batchBegin),
(System.currentTimeMillis() - esBatchBegin),
esBulkRequest.numberOfActions(),
mapping.get_index());
mapping.getIndex());
}
batchBegin = System.currentTimeMillis();
esBulkRequest.resetBulk();
@@ -180,12 +180,12 @@ public class ESEtlService extends AbstractEtlService {
(System.currentTimeMillis() - batchBegin),
(System.currentTimeMillis() - esBatchBegin),
esBulkRequest.numberOfActions(),
mapping.get_index());
mapping.getIndex());
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
errMsg.add(mapping.getIndex() + " etl failed! ==>" + e.getMessage());
throw new RuntimeException(e);
}
return count;

70
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java
View File

@@ -66,19 +66,19 @@ public class ES6xTemplate implements ESTemplate {
@Override
public void insert(ESSyncConfig.ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
if (mapping.get_id() != null) {
if (mapping.getId() != null) {
String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) {
ESUpdateRequest updateRequest = esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest updateRequest = esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) {
updateRequest.setRouting(parentVal);
}
getBulk().add(updateRequest);
} else {
ESIndexRequest indexRequest = esConnection.new ES6xIndexRequest(mapping.get_index(),
mapping.get_type(),
ESIndexRequest indexRequest = esConnection.new ES6xIndexRequest(mapping.getIndex(),
mapping.getType(),
pkVal.toString()).setSource(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) {
indexRequest.setRouting(parentVal);
@@ -87,13 +87,13 @@ public class ES6xTemplate implements ESTemplate {
}
commitBulk();
} else {
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
hit.getId()).setDoc(esFieldData);
getBulk().add(esUpdateRequest);
commitBulk();
@@ -150,19 +150,19 @@ public class ES6xTemplate implements ESTemplate {
@Override
public void delete(ESSyncConfig.ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
if (mapping.get_id() != null) {
ESDeleteRequest esDeleteRequest = this.esConnection.new ES6xDeleteRequest(mapping.get_index(),
mapping.get_type(),
if (mapping.getId() != null) {
ESDeleteRequest esDeleteRequest = this.esConnection.new ES6xDeleteRequest(mapping.getIndex(),
mapping.getType(),
pkVal.toString());
getBulk().add(esDeleteRequest);
commitBulk();
} else {
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
hit.getId()).setDoc(esFieldData);
getBulk().add(esUpdateRequest);
commitBulk();
@@ -207,7 +207,7 @@ public class ES6xTemplate implements ESTemplate {
public Object getESDataFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet,
Map<String, Object> esFieldData) throws SQLException {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
@@ -216,7 +216,7 @@ public class ES6xTemplate implements ESTemplate {
resultIdVal = value;
}
if (!fieldItem.getFieldName().equals(mapping.get_id())
if (!fieldItem.getFieldName().equals(mapping.getId())
&& !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
}
@@ -231,7 +231,7 @@ public class ES6xTemplate implements ESTemplate {
@Override
public Object getIdValFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet) throws SQLException {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
@@ -248,7 +248,7 @@ public class ES6xTemplate implements ESTemplate {
public Object getESDataFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
Map<String, Object> esFieldData) throws SQLException {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
if (fieldItem.getFieldName().equals(idFieldName)) {
@@ -294,7 +294,7 @@ public class ES6xTemplate implements ESTemplate {
public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping, Map<String, Object> dmlData,
Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
@@ -304,7 +304,7 @@ public class ES6xTemplate implements ESTemplate {
resultIdVal = value;
}
if (!fieldItem.getFieldName().equals(mapping.get_id())
if (!fieldItem.getFieldName().equals(mapping.getId())
&& !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
}
@@ -319,7 +319,7 @@ public class ES6xTemplate implements ESTemplate {
public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping,String owner, Map<String, Object> dmlData,
Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
@@ -353,19 +353,19 @@ public class ES6xTemplate implements ESTemplate {
}
private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
if (mapping.get_id() != null) {
if (mapping.getId() != null) {
String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) {
esUpdateRequest.setRouting(parentVal);
}
getBulk().add(esUpdateRequest);
} else {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
pkVal.toString()).setDoc(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) {
esUpdateRequest.setRouting(parentVal);
@@ -373,12 +373,12 @@ public class ES6xTemplate implements ESTemplate {
getBulk().add(esUpdateRequest);
}
} else {
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
mapping.getType(),
hit.getId()).setDoc(esFieldData);
getBulk().add(esUpdateRequest);
}
@@ -394,15 +394,15 @@ public class ES6xTemplate implements ESTemplate {
*/
@SuppressWarnings("unchecked")
private String getEsType(ESMapping mapping, String fieldName) {
String key = mapping.get_index() + "-" + mapping.get_type();
String key = mapping.getIndex() + "-" + mapping.getType();
Map<String, String> fieldType = esFieldTypes.get(key);
if (fieldType != null) {
return fieldType.get(fieldName);
} else {
MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index(), mapping.get_type());
MappingMetaData mappingMetaData = esConnection.getMapping(mapping.getIndex(), mapping.getType());
if (mappingMetaData == null) {
throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.getIndex());
}
fieldType = new LinkedHashMap<>();

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ConfigLoadTest.java
View File

@@ -29,9 +29,9 @@ public class ConfigLoadTest {
Assert.assertNotNull(config);
Assert.assertEquals("defaultDS", config.getDataSourceKey());
ESSyncConfig.ESMapping esMapping = config.getEsMapping();
Assert.assertEquals("mytest_user", esMapping.get_index());
Assert.assertEquals("_doc", esMapping.get_type());
Assert.assertEquals("id", esMapping.get_id());
Assert.assertEquals("mytest_user", esMapping.getIndex());
Assert.assertEquals("_doc", esMapping.getType());
Assert.assertEquals("id", esMapping.getId());
Assert.assertNotNull(esMapping.getSql());
// Map<String, List<ESSyncConfig>> dbTableEsSyncConfig =