chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

modify

Browse Source
This commit is contained in:
mcy
2018-10-31 18:47:16 +08:00
parent 94a3cfa9de
commit 14e094b522
7 changed files with 110 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java
View File

@@ -23,7 +23,7 @@ public class SchemaItem {
this.isAllFieldsSimple();
aliasTableItems.values().forEach(tableItem -> {
tableItem.getRelationTableFields();
tableItem.getRelationKeyFieldItems();
// tableItem.getRelationKeyFieldItems();
tableItem.getRelationSelectFieldItems();
});
}
@@ -145,8 +145,9 @@ public class SchemaItem {
private boolean main;
private boolean subQuery;
private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段
private volatile List<FieldItem> relationKeyFieldItems; // 关联条件字段在select中的对应字段
private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段对应主表查询字段
// private volatile List<FieldItem> relationKeyFieldItems; //
// 关联条件字段在select中的对应字段
private volatile List<FieldItem> relationSelectFieldItems; // 子表所在主表的查询字段
public TableItem(SchemaItem schemaItem){
@@ -267,34 +268,36 @@ public class SchemaItem {
return relationTableFields;
}
public List<FieldItem> getRelationKeyFieldItems() {
if (relationKeyFieldItems == null) {
synchronized (SchemaItem.class) {
if (relationKeyFieldItems == null) {
relationKeyFieldItems = new ArrayList<>();
getRelationFields().forEach(relationFieldsPair -> {
FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
.get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName());
if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
relationKeyFieldItems.addAll(selectFieldItem);
} else {
FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
selectFieldItem = getSchemaItem().getColumnFields()
.get(rightFieldItem.getOwner() + "." + rightFieldItem.getColumn().getColumnName());
if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
relationKeyFieldItems.addAll(selectFieldItem);
} else {
throw new UnsupportedOperationException(
"Relation condition column must in select columns.");
}
}
});
}
}
}
return relationKeyFieldItems;
}
// public List<FieldItem> getRelationKeyFieldItems() {
// if (relationKeyFieldItems == null) {
// synchronized (SchemaItem.class) {
// if (relationKeyFieldItems == null) {
// relationKeyFieldItems = new ArrayList<>();
// getRelationFields().forEach(relationFieldsPair -> {
// FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
// List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
// .get(leftFieldItem.getOwner() + "." +
// leftFieldItem.getColumn().getColumnName());
// if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
// relationKeyFieldItems.addAll(selectFieldItem);
// } else {
// FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
// selectFieldItem = getSchemaItem().getColumnFields()
// .get(rightFieldItem.getOwner() + "." +
// rightFieldItem.getColumn().getColumnName());
// if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
// relationKeyFieldItems.addAll(selectFieldItem);
// } else {
// throw new UnsupportedOperationException(
// "Relation condition column must in select columns.");
// }
// }
// });
// }
// }
// }
// return relationKeyFieldItems;
// }
public List<FieldItem> getRelationSelectFieldItems() {
if (relationSelectFieldItems == null) {

41
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -190,41 +190,46 @@ public class ESSyncService {
if (allFieldsSimple) {
// 不是子查询
if (!tableItem.isSubQuery()) {
Map<String, Object> esFieldData = new LinkedHashMap<>();
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
Object value = esTemplate
.getValFromData(mapping, data, fieldItem.getColumn().getColumnName());
esFieldData.put(fieldItem.getFieldName(), value);
}
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields()
.entrySet()) {
Object value = esTemplate
.getValFromData(mapping, data, entry.getKey().getColumn().getColumnName());
for (FieldItem fieldItem : entry.getValue()) {
queryBuilder.must(QueryBuilders.termsQuery(fieldItem.getFieldName(), value));
String fieldName = fieldItem.getFieldName();
// 判断是否是主键
if (fieldName.equals(mapping.get_id())) {
fieldName = "_id";
}
queryBuilder.must(QueryBuilders.termsQuery(fieldName, value));
}
}
//
// for (FieldItem fieldItem : tableItem.getRelationTableFields()) {
// Object value = esTemplate
// .getValFromData(mapping, data, fieldItem.getColumn().getColumnName());
//
// }
// if (logger.isDebugEnabled()) {
// logger.debug("从表insert, 且均为简单字段，对es进行update_by_query, queryBuilder: {},
// esFieldData:{}", queryBuilder.toString(), esFieldData);
// }
if (logger.isDebugEnabled()) {
logger.trace(
"Join table update es index by foreign key, destination:{}, table: {}, index: {}",
config.getDestination(),
dml.getTable(),
mapping.get_index());
}
boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData);
// if (!result) {
// logger.error("从表insert，均为简单字段，直接对es进行update_by_query, es更新存在错误, table: {},
// index: {}, dml: {}", dml.getTable(), config.getEsSyn().getIndex(), dml);
// }
if (!result) {
logger.error(
"Join table update es index by foreign key error, destination:{}, table: {}, index: {}",
config.getDestination(),
dml.getTable(),
mapping.get_index());
}
} else {
// TODO
}
// TODO
} else {
// TODO 查询总sql
}

4
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java
View File

@@ -264,7 +264,7 @@ public class ESTemplate {
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.warn(itemResponse.getFailureMessage());
} else {
logger.error("ES 同步数据错误 {}", itemResponse.getFailureMessage());
logger.error("ES sync commit error: {}", itemResponse.getFailureMessage());
}
}
}
@@ -314,7 +314,7 @@ public class ESTemplate {
value = resultSet.getByte(fieldName);
}
}
return value;
return ESSyncUtil.typeConvert(value, esType);
}
public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,