chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix sync order in RdbMirrorDbSyncService (#3928)

Browse Source
This commit is contained in:
He Wang
2022-05-13 14:19:40 +08:00
committed by GitHub
parent 67e85d201c
commit bd1f91cd9c
2 changed files with 55 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java
View File

@@ -17,7 +17,6 @@ import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONWriter.Feature; import com.alibaba.fastjson2.JSONWriter.Feature;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig;
import com.alibaba.otter.canal.client.adapter.rdb.support.SingleDml;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
/** /**
@@ -45,7 +44,7 @@ public class RdbMirrorDbSyncService {
/** /**
* 批量同步方法 * 批量同步方法
* *
* @param dmls 批量 DML * @param dmls 批量 DML，包含DDL
*/ */
public void sync(List<Dml> dmls) { public void sync(List<Dml> dmls) {
List<Dml> dmlList = new ArrayList<>(); List<Dml> dmlList = new ArrayList<>();
@@ -66,6 +65,10 @@ public class RdbMirrorDbSyncService {
} }
if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
// 确保执行DDL前DML已执行完
syncDml(dmlList);
dmlList.clear();
// DDL // DDL
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("DDL: {}", JSON.toJSONString(dml, Feature.WriteNulls)); logger.debug("DDL: {}", JSON.toJSONString(dml, Feature.WriteNulls));
@@ -79,38 +82,32 @@ public class RdbMirrorDbSyncService {
dmlList.add(dml); dmlList.add(dml);
} }
} }
if (!dmlList.isEmpty()) { syncDml(dmlList);
rdbSyncService.sync(dmlList, dml -> { }
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase());
if (mirrorDbConfig == null) {
return false;
}
String table = dml.getTable();
MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
if (config == null) { /**
return false; * 批量同步Dml
} *
// 是否区分大小写 * @param dmlList Dml列表，不包含DDL
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive(); */
if (config.getConcurrent()) { private void syncDml(List<Dml> dmlList) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive); if (dmlList == null || dmlList.isEmpty()) {
singleDmls.forEach(singleDml -> { return;
int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData());
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
rdbSyncService.getDmlsPartition()[hash].add(syncItem);
});
} else {
int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
rdbSyncService.getDmlsPartition()[hash].add(syncItem);
});
}
return true;
});
} }
rdbSyncService.sync(dmlList, dml -> {
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase());
if (mirrorDbConfig == null) {
return false;
}
String table = dml.getTable();
MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
if (config == null) {
return false;
}
rdbSyncService.appendDmlPartition(config, dml);
return true;
});
} }
/** /**

42
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -177,28 +177,38 @@ public class RdbSyncService {
} }
for (MappingConfig config : configMap.values()) { for (MappingConfig config : configMap.values()) {
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive(); appendDmlPartition(config, dml);
if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
int hash = pkHash(config.getDbMapping(), singleDml.getData());
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
} else {
int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
}
} }
return true; return true;
} }
} ); } );
} }
/**
* 将Dml加入 {@link #dmlsPartition}
*
* @param config 表映射配置
* @param dml Dml对象
*/
public void appendDmlPartition(MappingConfig config, Dml dml) {
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
int hash = pkHash(config.getDbMapping(), singleDml.getData());
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
} else {
int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
}
}
/** /**
* 单条 dml 同步 * 单条 dml 同步
* *