chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix bug #2023 and code optimize (#2167)

Browse Source 
* 减少等待刷新状态时间

* fix bug #2023
This commit is contained in:
rewerma
2019-09-12 08:47:22 +08:00
committed by agapple
parent 803565456f
commit 31bb85bea3
5 changed files with 79 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java
View File

@@ -109,13 +109,15 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
}));
}
futures.forEach(f -> {
for (Future<Void> f :futures){
try {
f.get(3, TimeUnit.SECONDS);
} catch (TimeoutException | InterruptedException | ExecutionException e) {
} catch (InterruptedException | ExecutionException e) {
// ignore
} catch (TimeoutException e){
break;
}
}
});
return pager;
}

8
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java
View File

@@ -154,13 +154,15 @@ public class NodeServerServiceImpl implements NodeServerService {
return !status;
}));
}
futures.forEach(f -> {
for (Future<Boolean> f :futures){
try {
f.get(3, TimeUnit.SECONDS);
} catch (TimeoutException | InterruptedException | ExecutionException e) {
} catch (InterruptedException | ExecutionException e) {
// ignore
} catch (TimeoutException e){
break;
}
}
});
return pager;
}

17
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -203,7 +203,7 @@ public class ESSyncService {
if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
// ------单表 & 所有字段都为简单字段------
singleTableSimpleFiledUpdate(config, dml, data, old);
singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
} else {
// ------主表 查询sql来更新------
if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
@@ -219,6 +219,7 @@ public class ESSyncService {
boolean allUpdateFieldSimple = true;
out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
for (ColumnItem columnItem : fieldItem.getColumnItems()) {
if (schemaItem.getMainTable().getAlias().equals(columnItem.getOwner())) { // 如果是主表的字段
if (old.containsKey(columnItem.getColumnName())) {
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
allUpdateFieldSimple = false;
@@ -227,6 +228,7 @@ public class ESSyncService {
}
}
}
}
// 不支持主键更新!!
@@ -257,7 +259,7 @@ public class ESSyncService {
// 判断主键和所更新的字段是否全为简单字段
if (idFieldSimple && allUpdateFieldSimple && !fkChanged) {
singleTableSimpleFiledUpdate(config, dml, data, old);
singleTableSimpleFiledUpdate(config,schemaItem.getMainTable().getAlias(), dml, data, old);
} else {
mainTableUpdate(config, dml, data, old);
}
@@ -674,14 +676,14 @@ public class ESSyncService {
private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old,
TableItem tableItem) {
ESMapping mapping = config.getEsMapping();
//防止最后出现groupby 导致sql解析异常
// 防止最后出现groupby 导致sql解析异常
String[] sqlSplit = mapping.getSql().split("GROUP\\ BY(?!(.*)ON)");
String sqlNoWhere = sqlSplit[0];
String sqlGroupBy = "";
if(sqlSplit.length > 1){
sqlGroupBy = "GROUP BY "+ sqlSplit[1];
if (sqlSplit.length > 1) {
sqlGroupBy = "GROUP BY " + sqlSplit[1];
}
StringBuilder sql = new StringBuilder(sqlNoWhere + " WHERE ");
@@ -781,16 +783,17 @@ public class ESSyncService {
* 单表简单字段update
*
* @param config es配置
* @param owner 所属表
* @param dml dml信息
* @param data 单行data数据
* @param old 单行old数据
*/
private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data,
private void singleTableSimpleFiledUpdate(ESSyncConfig config, String owner, Dml dml, Map<String, Object> data,
Map<String, Object> old) {
ESMapping mapping = config.getEsMapping();
Map<String, Object> esFieldData = new LinkedHashMap<>();
Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData);
Object idVal = esTemplate.getESDataFromDmlData(mapping, owner, data, old, esFieldData);
if (logger.isTraceEnabled()) {
logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}",

3
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESConnection.java
View File

@@ -61,7 +61,8 @@ public class ESConnection {
this.mode = mode;
if (mode == ESClientMode.TRANSPORT) {
Settings.Builder settingBuilder = Settings.builder();
properties.forEach(settingBuilder::put);
// properties.forEach(settingBuilder::put);
settingBuilder.put("cluster.name", properties.get("properties"));
Settings settings = settingBuilder.build();
transportClient = new PreBuiltTransportClient(settings);
for (String host : hosts) {

13
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java
View File

@@ -45,7 +45,7 @@ public class ESTemplate {
private ESBulkRequest esBulkRequest;
public ESTemplate(ESConnection esConnection) {
public ESTemplate(ESConnection esConnection){
this.esConnection = esConnection;
this.esBulkRequest = this.esConnection.new ESBulkRequest();
}
@@ -394,17 +394,22 @@ public class ESTemplate {
* 将dml的data, old转换为es的data
*
* @param mapping 配置mapping
* @param owner 所属表
* @param dmlData dml data
* @param esFieldData es data
* @return 返回 id 值
*/
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> dmlOld,
Map<String, Object> esFieldData) {
public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
if (!columnItem.getOwner().equals(owner)) {
continue;
}
String columnName = columnItem.getColumnName();
if (fieldItem.getFieldName().equals(idFieldName)) {
resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);