chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed hash mode

Browse Source
This commit is contained in:
agapple
2018-12-11 20:41:21 +08:00
parent 0197482618
commit 35e0e882c1
1 changed files with 42 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
View File

@@ -41,11 +41,20 @@ public class MQMessageUtils {
PartitionData data = new PartitionData(); PartitionData data = new PartitionData();
int i = pkHashConfig.lastIndexOf(":"); int i = pkHashConfig.lastIndexOf(":");
if (i > 0) { if (i > 0) {
data.pkNames = Lists.newArrayList(StringUtils.split(pkHashConfig.substring(i + 1), String pkStr = pkHashConfig.substring(i + 1);
'^')); if (pkStr.equalsIgnoreCase("$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
'^'));
}
pkHashConfig = pkHashConfig.substring(0, pkHashConfig = pkHashConfig.substring(0,
i); i);
} else {
data.hashMode.tableHash = true;
} }
if (!isWildCard(pkHashConfig)) { if (!isWildCard(pkHashConfig)) {
data.simpleName = pkHashConfig; data.simpleName = pkHashConfig;
} else { } else {
@@ -107,24 +116,23 @@ public class MQMessageUtils {
if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) { if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
String database = entry.getHeader().getSchemaName(); String database = entry.getHeader().getSchemaName();
String table = entry.getHeader().getTableName(); String table = entry.getHeader().getTableName();
List<String> pkNames = getParitionHashColumns(database + "." + table, pkHashConfigs); HashMode hashMode = getParitionHashColumns(database + "." + table, pkHashConfigs);
if (hashMode == null) {
if (pkNames == null) {
// 如果都没有匹配，发送到第一个分区 // 如果都没有匹配，发送到第一个分区
partitionEntries[0].add(entry); partitionEntries[0].add(entry);
} else { } else {
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) { for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
int hashCode = table.hashCode(); int hashCode = table.hashCode();
if (pkNames.isEmpty()) { if (hashMode.autoPkHash) {
// isEmpty use default pkNames // isEmpty use default pkNames
for (CanalEntry.Column column : rowData.getAfterColumnsList()) { for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
if (column.getIsKey()) { if (column.getIsKey()) {
hashCode = hashCode ^ column.getValue().hashCode(); hashCode = hashCode ^ column.getValue().hashCode();
} }
} }
} else { } else if (!hashMode.tableHash) {
for (CanalEntry.Column column : rowData.getAfterColumnsList()) { for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
if (checkPkNamesHasContain(pkNames, column.getName())) { if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
hashCode = hashCode ^ column.getValue().hashCode(); hashCode = hashCode ^ column.getValue().hashCode();
} }
} }
@@ -299,24 +307,27 @@ public class MQMessageUtils {
if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) { if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
String database = flatMessage.getDatabase(); String database = flatMessage.getDatabase();
String table = flatMessage.getTable(); String table = flatMessage.getTable();
List<String> pkNames = getParitionHashColumns(database + "." + table, pkHashConfigs); HashMode hashMode = getParitionHashColumns(database + "." + table, pkHashConfigs);
if (pkNames == null) { if (hashMode == null) {
// 如果都没有匹配，发送到第一个分区 // 如果都没有匹配，发送到第一个分区
partitionMessages[0] = flatMessage; partitionMessages[0] = flatMessage;
} else { } else {
if (pkNames.isEmpty()) { List<String> pkNames = hashMode.pkNames;
if (hashMode.autoPkHash) {
pkNames = flatMessage.getPkNames(); pkNames = flatMessage.getPkNames();
} }
int hashCode = table.hashCode(); int hashCode = table.hashCode();
int idx = 0; int idx = 0;
for (Map<String, String> row : flatMessage.getData()) { for (Map<String, String> row : flatMessage.getData()) {
for (String pkName : pkNames) { if (!hashMode.tableHash) {
String value = row.get(pkName); for (String pkName : pkNames) {
if (value == null) { String value = row.get(pkName);
value = ""; if (value == null) {
value = "";
}
hashCode = hashCode ^ value.hashCode();
} }
hashCode = hashCode ^ value.hashCode();
} }
int pkHash = Math.abs(hashCode) % partitionsNum; int pkHash = Math.abs(hashCode) % partitionsNum;
@@ -362,16 +373,20 @@ public class MQMessageUtils {
/** /**
* match return List , not match return null * match return List , not match return null
*/ */
public static List<String> getParitionHashColumns(String name, String pkHashConfigs) { public static HashMode getParitionHashColumns(String name, String pkHashConfigs) {
if (StringUtils.isEmpty(pkHashConfigs)) {
return null;
}
List<PartitionData> datas = partitionDatas.get(pkHashConfigs); List<PartitionData> datas = partitionDatas.get(pkHashConfigs);
for (PartitionData data : datas) { for (PartitionData data : datas) {
if (data.simpleName != null) { if (data.simpleName != null) {
if (data.simpleName.equalsIgnoreCase(name)) { if (data.simpleName.equalsIgnoreCase(name)) {
return data.pkNames; return data.hashMode;
} }
} else { } else {
if (data.regexFilter.filter(name)) { if (data.regexFilter.filter(name)) {
return data.pkNames; return data.hashMode;
} }
} }
} }
@@ -399,7 +414,14 @@ public class MQMessageUtils {
public String simpleName; public String simpleName;
public AviaterRegexFilter regexFilter; public AviaterRegexFilter regexFilter;
public List<String> pkNames = Lists.newArrayList(); public HashMode hashMode = new HashMode();
}
public static class HashMode {
public boolean autoPkHash = false;
public boolean tableHash = false;
public List<String> pkNames = Lists.newArrayList();
} }
} }