chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

根据主键Hash增加正则表达式支持，pk通配符指定自动获取支持

Browse Source
This commit is contained in:
machey
2018-12-10 00:09:49 +08:00
parent ac738dae05
commit 03ee9c7ecf
4 changed files with 123 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java
View File

@@ -1,12 +1,10 @@
package com.alibaba.otter.canal.protocol;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import com.google.protobuf.ByteString;
@@ -33,8 +31,7 @@ public class FlatMessage implements Serializable {
private List<Map<String, String>> data;
private List<Map<String, String>> old;
public FlatMessage(){
}
private transient Message message; // 所属message
public FlatMessage(long id){
this.id = id;
@@ -136,6 +133,14 @@ public class FlatMessage implements Serializable {
this.es = es;
}
public Message getMessage() {
return message;
}
public void setMessage(Message message) {
this.message = message;
}
/**
* 将Message转换为FlatMessage
*
@@ -187,6 +192,7 @@ public class FlatMessage implements Serializable {
flatMessage.setEs(entry.getHeader().getExecuteTime());
flatMessage.setTs(System.currentTimeMillis());
flatMessage.setSql(rowChange.getSql());
flatMessage.setMessage(message);
if (!rowChange.getIsDdl()) {
Map<String, Integer> sqlType = new LinkedHashMap<>();
@@ -269,70 +275,116 @@ public class FlatMessage implements Serializable {
*
* @param flatMessage flatMessage
* @param partitionsNum 分区数量
* @param pkHashConfig hash映射
* @param pkHashConfigs hash映射
* @return 拆分后的flatMessage数组
*/
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum,
Map<String, String> pkHashConfig) {
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum, String pkHashConfigs) {
if (partitionsNum == null) {
partitionsNum = 1;
}
FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
String pk = pkHashConfig.get(flatMessage.getDatabase() + "." + flatMessage.getTable());
if (pk == null || flatMessage.getIsDdl()) {
if (flatMessage.getIsDdl()) {
partitionMessages[0] = flatMessage;
} else {
if (flatMessage.getData() != null) {
int idx = 0;
for (Map<String, String> row : flatMessage.getData()) {
Map<String, String> o = null;
if (flatMessage.getOld() != null) {
o = flatMessage.getOld().get(idx);
}
String value;
// 如果old中有pk值说明主键有修改, 以旧的主键值hash为准
if (o != null && o.containsKey(pk)) {
value = o.get(pk);
} else {
value = row.get(pk);
}
if (value == null) {
value = "";
}
int hash = value.hashCode();
int pkHash = Math.abs(hash) % partitionsNum;
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
pkHash = Math.abs(pkHash);
String database = flatMessage.getDatabase();
String table = flatMessage.getTable();
FlatMessage flatMessageTmp = partitionMessages[pkHash];
if (flatMessageTmp == null) {
flatMessageTmp = new FlatMessage(flatMessage.getId());
partitionMessages[pkHash] = flatMessageTmp;
flatMessageTmp.setDatabase(flatMessage.getDatabase());
flatMessageTmp.setTable(flatMessage.getTable());
flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
flatMessageTmp.setType(flatMessage.getType());
flatMessageTmp.setSql(flatMessage.getSql());
flatMessageTmp.setSqlType(flatMessage.getSqlType());
flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
flatMessageTmp.setEs(flatMessage.getEs());
flatMessageTmp.setTs(flatMessage.getTs());
String pk = null;
boolean isMatch = false;
String[] pkHashConfigArray = StringUtils.split(pkHashConfigs, ",");
for (String pkHashConfig : pkHashConfigArray) {
int i = pkHashConfig.lastIndexOf(".");
if (!pkHashConfig.endsWith(".$pk$")) {
// 如果指定了主键
pk = pkHashConfig.substring(i + 1);
}
List<Map<String, String>> data = flatMessageTmp.getData();
if (data == null) {
data = new ArrayList<>();
flatMessageTmp.setData(data);
pkHashConfig = pkHashConfig.substring(0, i);
isMatch = Pattern.matches(pkHashConfig, database + "." + table);
if (isMatch) {
break;
}
data.add(row);
if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
List<Map<String, String>> old = flatMessageTmp.getOld();
if (old == null) {
old = new ArrayList<>();
flatMessageTmp.setOld(old);
}
if (!isMatch) {
// 如果都没有匹配，发送到第一个分区
partitionMessages[0] = flatMessage;
} else {
if (pk == null) {
// 如果未指定主键(通配符主键)，从原生message中取主键字段
CanalEntry.Entry entry = flatMessage.getMessage().getEntries().get(0);
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
CanalEntry.RowData rowData = rowChange.getRowDatasList().get(0);
for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
if (column.getIsKey()) {
pk = column.getName();
break;
}
}
}
if (pk == null || !flatMessage.getData().get(0).containsKey(pk)) {
// 如果都没有匹配的主键，发送到第一个分区
partitionMessages[0] = flatMessage;
} else {
int idx = 0;
for (Map<String, String> row : flatMessage.getData()) {
Map<String, String> o = null;
if (flatMessage.getOld() != null) {
o = flatMessage.getOld().get(idx);
}
String value;
// 如果old中有pk值说明主键有修改, 以旧的主键值hash为准
if (o != null && o.containsKey(pk)) {
value = o.get(pk);
} else {
value = row.get(pk);
}
if (value == null) {
value = "";
}
int hash = value.hashCode();
int pkHash = Math.abs(hash) % partitionsNum;
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
pkHash = Math.abs(pkHash);
FlatMessage flatMessageTmp = partitionMessages[pkHash];
if (flatMessageTmp == null) {
flatMessageTmp = new FlatMessage(flatMessage.getId());
partitionMessages[pkHash] = flatMessageTmp;
flatMessageTmp.setDatabase(flatMessage.getDatabase());
flatMessageTmp.setTable(flatMessage.getTable());
flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
flatMessageTmp.setType(flatMessage.getType());
flatMessageTmp.setSql(flatMessage.getSql());
flatMessageTmp.setSqlType(flatMessage.getSqlType());
flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
flatMessageTmp.setEs(flatMessage.getEs());
flatMessageTmp.setTs(flatMessage.getTs());
}
List<Map<String, String>> data = flatMessageTmp.getData();
if (data == null) {
data = new ArrayList<>();
flatMessageTmp.setData(data);
}
data.add(row);
if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
List<Map<String, String>> old = flatMessageTmp.getOld();
if (old == null) {
old = new ArrayList<>();
flatMessageTmp.setOld(old);
}
old.add(flatMessage.getOld().get(idx));
}
idx++;
}
old.add(flatMessage.getOld().get(idx));
}
idx++;
}
}
}