chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix bug #2023 and code optimize (#2167)

Browse Source 
* 减少等待刷新状态时间

* fix bug #2023
This commit is contained in:
rewerma
2019-09-12 08:47:22 +08:00
committed by agapple
parent 803565456f
commit 31bb85bea3
5 changed files with 79 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java
View File

@@ -109,13 +109,15 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
})); }));
} }
futures.forEach(f -> { for (Future<Void> f :futures){
try { try {
f.get(3, TimeUnit.SECONDS); f.get(3, TimeUnit.SECONDS);
} catch (TimeoutException | InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
// ignore // ignore
} catch (TimeoutException e){
break;
} }
}); }
return pager; return pager;
} }

8
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java
View File

@@ -154,13 +154,15 @@ public class NodeServerServiceImpl implements NodeServerService {
return !status; return !status;
})); }));
} }
futures.forEach(f -> { for (Future<Boolean> f :futures){
try { try {
f.get(3, TimeUnit.SECONDS); f.get(3, TimeUnit.SECONDS);
} catch (TimeoutException | InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
// ignore // ignore
} catch (TimeoutException e){
break;
} }
}); }
return pager; return pager;
} }

25
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -203,7 +203,7 @@ public class ESSyncService {
if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
// ------单表 & 所有字段都为简单字段------ // ------单表 & 所有字段都为简单字段------
singleTableSimpleFiledUpdate(config, dml, data, old); singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
} else { } else {
// ------主表 查询sql来更新------ // ------主表 查询sql来更新------
if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
@@ -219,10 +219,12 @@ public class ESSyncService {
boolean allUpdateFieldSimple = true; boolean allUpdateFieldSimple = true;
out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
for (ColumnItem columnItem : fieldItem.getColumnItems()) { for (ColumnItem columnItem : fieldItem.getColumnItems()) {
if (old.containsKey(columnItem.getColumnName())) { if (schemaItem.getMainTable().getAlias().equals(columnItem.getOwner())) { // 如果是主表的字段
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { if (old.containsKey(columnItem.getColumnName())) {
allUpdateFieldSimple = false; if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
break out; allUpdateFieldSimple = false;
break out;
}
} }
} }
} }
@@ -257,7 +259,7 @@ public class ESSyncService {
// 判断主键和所更新的字段是否全为简单字段 // 判断主键和所更新的字段是否全为简单字段
if (idFieldSimple && allUpdateFieldSimple && !fkChanged) { if (idFieldSimple && allUpdateFieldSimple && !fkChanged) {
singleTableSimpleFiledUpdate(config, dml, data, old); singleTableSimpleFiledUpdate(config,schemaItem.getMainTable().getAlias(), dml, data, old);
} else { } else {
mainTableUpdate(config, dml, data, old); mainTableUpdate(config, dml, data, old);
} }
@@ -674,14 +676,14 @@ public class ESSyncService {
private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old, private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old,
TableItem tableItem) { TableItem tableItem) {
ESMapping mapping = config.getEsMapping(); ESMapping mapping = config.getEsMapping();
//防止最后出现groupby 导致sql解析异常 // 防止最后出现groupby 导致sql解析异常
String[] sqlSplit = mapping.getSql().split("GROUP\\ BY(?!(.*)ON)"); String[] sqlSplit = mapping.getSql().split("GROUP\\ BY(?!(.*)ON)");
String sqlNoWhere = sqlSplit[0]; String sqlNoWhere = sqlSplit[0];
String sqlGroupBy = ""; String sqlGroupBy = "";
if(sqlSplit.length > 1){ if (sqlSplit.length > 1) {
sqlGroupBy = "GROUP BY "+ sqlSplit[1]; sqlGroupBy = "GROUP BY " + sqlSplit[1];
} }
StringBuilder sql = new StringBuilder(sqlNoWhere + " WHERE "); StringBuilder sql = new StringBuilder(sqlNoWhere + " WHERE ");
@@ -781,16 +783,17 @@ public class ESSyncService {
* 单表简单字段update * 单表简单字段update
* *
* @param config es配置 * @param config es配置
* @param owner 所属表
* @param dml dml信息 * @param dml dml信息
* @param data 单行data数据 * @param data 单行data数据
* @param old 单行old数据 * @param old 单行old数据
*/ */
private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data, private void singleTableSimpleFiledUpdate(ESSyncConfig config, String owner, Dml dml, Map<String, Object> data,
Map<String, Object> old) { Map<String, Object> old) {
ESMapping mapping = config.getEsMapping(); ESMapping mapping = config.getEsMapping();
Map<String, Object> esFieldData = new LinkedHashMap<>(); Map<String, Object> esFieldData = new LinkedHashMap<>();
Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData); Object idVal = esTemplate.getESDataFromDmlData(mapping, owner, data, old, esFieldData);
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}", logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}",

3
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESConnection.java
View File

@@ -61,7 +61,8 @@ public class ESConnection {
this.mode = mode; this.mode = mode;
if (mode == ESClientMode.TRANSPORT) { if (mode == ESClientMode.TRANSPORT) {
Settings.Builder settingBuilder = Settings.builder(); Settings.Builder settingBuilder = Settings.builder();
properties.forEach(settingBuilder::put); // properties.forEach(settingBuilder::put);
settingBuilder.put("cluster.name", properties.get("properties"));
Settings settings = settingBuilder.build(); Settings settings = settingBuilder.build();
transportClient = new PreBuiltTransportClient(settings); transportClient = new PreBuiltTransportClient(settings);
for (String host : hosts) { for (String host : hosts) {

101
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java
View File

@@ -37,15 +37,15 @@ import com.alibaba.otter.canal.client.adapter.support.Util;
*/ */
public class ESTemplate { public class ESTemplate {
private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class);
private static final int MAX_BATCH_SIZE = 1000; private static final int MAX_BATCH_SIZE = 1000;
private ESConnection esConnection; private ESConnection esConnection;
private ESBulkRequest esBulkRequest; private ESBulkRequest esBulkRequest;
public ESTemplate(ESConnection esConnection) { public ESTemplate(ESConnection esConnection){
this.esConnection = esConnection; this.esConnection = esConnection;
this.esBulkRequest = this.esConnection.new ESBulkRequest(); this.esBulkRequest = this.esConnection.new ESBulkRequest();
} }
@@ -61,8 +61,8 @@ public class ESTemplate {
/** /**
* 插入数据 * 插入数据
* *
* @param mapping 配置对象 * @param mapping 配置对象
* @param pkVal 主键值 * @param pkVal 主键值
* @param esFieldData 数据Map * @param esFieldData 数据Map
*/ */
public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) { public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
@@ -70,16 +70,16 @@ public class ESTemplate {
String parentVal = (String) esFieldData.remove("$parent_routing"); String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) { if (mapping.isUpsert()) {
ESUpdateRequest updateRequest = esConnection.new ESUpdateRequest(mapping.get_index(), ESUpdateRequest updateRequest = esConnection.new ESUpdateRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) { if (StringUtils.isNotEmpty(parentVal)) {
updateRequest.setRouting(parentVal); updateRequest.setRouting(parentVal);
} }
getBulk().add(updateRequest); getBulk().add(updateRequest);
} else { } else {
ESIndexRequest indexRequest = esConnection.new ESIndexRequest(mapping.get_index(), ESIndexRequest indexRequest = esConnection.new ESIndexRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
pkVal.toString()).setSource(esFieldData); pkVal.toString()).setSource(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) { if (StringUtils.isNotEmpty(parentVal)) {
indexRequest.setRouting(parentVal); indexRequest.setRouting(parentVal);
} }
@@ -88,13 +88,13 @@ public class ESTemplate {
commitBulk(); commitBulk();
} else { } else {
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000); mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse(); SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) { for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(), ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
hit.getId()).setDoc(esFieldData); hit.getId()).setDoc(esFieldData);
getBulk().add(esUpdateRequest); getBulk().add(esUpdateRequest);
commitBulk(); commitBulk();
} }
@@ -105,8 +105,8 @@ public class ESTemplate {
/** /**
* 根据主键更新数据 * 根据主键更新数据
* *
* @param mapping 配置对象 * @param mapping 配置对象
* @param pkVal 主键值 * @param pkVal 主键值
* @param esFieldData 数据Map * @param esFieldData 数据Map
*/ */
public void update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) { public void update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
@@ -119,8 +119,8 @@ public class ESTemplate {
/** /**
* update by query * update by query
* *
* @param config 配置对象 * @param config 配置对象
* @param paramsTmp sql查询条件 * @param paramsTmp sql查询条件
* @param esFieldData 数据Map * @param esFieldData 数据Map
*/ */
public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) { public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
@@ -164,25 +164,25 @@ public class ESTemplate {
/** /**
* 通过主键删除数据 * 通过主键删除数据
* *
* @param mapping 配置对象 * @param mapping 配置对象
* @param pkVal 主键值 * @param pkVal 主键值
* @param esFieldData 数据Map * @param esFieldData 数据Map
*/ */
public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) { public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
if (mapping.get_id() != null) { if (mapping.get_id() != null) {
ESDeleteRequest esDeleteRequest = this.esConnection.new ESDeleteRequest(mapping.get_index(), ESDeleteRequest esDeleteRequest = this.esConnection.new ESDeleteRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
pkVal.toString()); pkVal.toString());
getBulk().add(esDeleteRequest); getBulk().add(esDeleteRequest);
commitBulk(); commitBulk();
} else { } else {
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000); mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse(); SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) { for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(), ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
hit.getId()).setDoc(esFieldData); hit.getId()).setDoc(esFieldData);
getBulk().add(esUpdateRequest); getBulk().add(esUpdateRequest);
commitBulk(); commitBulk();
} }
@@ -227,16 +227,16 @@ public class ESTemplate {
String parentVal = (String) esFieldData.remove("$parent_routing"); String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) { if (mapping.isUpsert()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(), ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) { if (StringUtils.isNotEmpty(parentVal)) {
esUpdateRequest.setRouting(parentVal); esUpdateRequest.setRouting(parentVal);
} }
getBulk().add(esUpdateRequest); getBulk().add(esUpdateRequest);
} else { } else {
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(), ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
pkVal.toString()).setDoc(esFieldData); pkVal.toString()).setDoc(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) { if (StringUtils.isNotEmpty(parentVal)) {
esUpdateRequest.setRouting(parentVal); esUpdateRequest.setRouting(parentVal);
} }
@@ -244,12 +244,12 @@ public class ESTemplate {
} }
} else { } else {
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000); mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse(); SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) { for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(), ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
mapping.get_type(), mapping.get_type(),
hit.getId()).setDoc(esFieldData); hit.getId()).setDoc(esFieldData);
getBulk().add(esUpdateRequest); getBulk().add(esUpdateRequest);
} }
} }
@@ -289,7 +289,7 @@ public class ESTemplate {
} }
if (!fieldItem.getFieldName().equals(mapping.get_id()) if (!fieldItem.getFieldName().equals(mapping.get_id())
&& !mapping.getSkips().contains(fieldItem.getFieldName())) { && !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
} }
} }
@@ -327,9 +327,9 @@ public class ESTemplate {
for (ColumnItem columnItem : fieldItem.getColumnItems()) { for (ColumnItem columnItem : fieldItem.getColumnItems()) {
if (dmlOld.containsKey(columnItem.getColumnName()) if (dmlOld.containsKey(columnItem.getColumnName())
&& !mapping.getSkips().contains(fieldItem.getFieldName())) { && !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
break; break;
} }
} }
@@ -361,8 +361,8 @@ public class ESTemplate {
/** /**
* 将dml的data转换为es的data * 将dml的data转换为es的data
* *
* @param mapping 配置mapping * @param mapping 配置mapping
* @param dmlData dml data * @param dmlData dml data
* @param esFieldData es data * @param esFieldData es data
* @return 返回 id 值 * @return 返回 id 值
*/ */
@@ -380,7 +380,7 @@ public class ESTemplate {
} }
if (!fieldItem.getFieldName().equals(mapping.get_id()) if (!fieldItem.getFieldName().equals(mapping.get_id())
&& !mapping.getSkips().contains(fieldItem.getFieldName())) { && !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
} }
} }
@@ -393,18 +393,23 @@ public class ESTemplate {
/** /**
* 将dml的data, old转换为es的data * 将dml的data, old转换为es的data
* *
* @param mapping 配置mapping * @param mapping 配置mapping
* @param dmlData dml data * @param owner 所属表
* @param dmlData dml data
* @param esFieldData es data * @param esFieldData es data
* @return 返回 id 值 * @return 返回 id 值
*/ */
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> dmlOld, public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
Map<String, Object> esFieldData) { Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
SchemaItem schemaItem = mapping.getSchemaItem(); SchemaItem schemaItem = mapping.getSchemaItem();
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
Object resultIdVal = null; Object resultIdVal = null;
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
if (!columnItem.getOwner().equals(owner)) {
continue;
}
String columnName = columnItem.getColumnName();
if (fieldItem.getFieldName().equals(idFieldName)) { if (fieldItem.getFieldName().equals(idFieldName)) {
resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
@@ -412,7 +417,7 @@ public class ESTemplate {
if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) { if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
} }
} }
@@ -433,9 +438,9 @@ public class ESTemplate {
Object parentVal; Object parentVal;
try { try {
parentVal = getValFromRS(mapping, parentVal = getValFromRS(mapping,
resultSet, resultSet,
parentFieldItem.getFieldName(), parentFieldItem.getFieldName(),
parentFieldItem.getFieldName()); parentFieldItem.getFieldName());
} catch (SQLException e) { } catch (SQLException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@@ -480,7 +485,7 @@ public class ESTemplate {
/** /**
* 获取es mapping中的属性类型 * 获取es mapping中的属性类型
* *
* @param mapping mapping配置 * @param mapping mapping配置
* @param fieldName 属性名 * @param fieldName 属性名
* @return 类型 * @return 类型
*/ */