chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

对原生message发送mq的分区支持

Browse Source
This commit is contained in:
machey
2018-12-10 01:03:18 +08:00
parent 03ee9c7ecf
commit 547cbf82c8
4 changed files with 240 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java
View File

@@ -14,24 +14,24 @@ import com.google.protobuf.ByteString;
*/
public class FlatMessage implements Serializable {
private static final long serialVersionUID = -3386650678735860050L;
private static final long serialVersionUID = -3386650678735860050L;
private long id;
private String database;
private String table;
private Boolean isDdl;
private String type;
private long id;
private String database;
private String table;
private Boolean isDdl;
private String type;
// binlog executeTime
private Long es;
private Long es;
// dml build timeStamp
private Long ts;
private String sql;
private Map<String, Integer> sqlType;
private Map<String, String> mysqlType;
private List<Map<String, String>> data;
private List<Map<String, String>> old;
private Long ts;
private String sql;
private Map<String, Integer> sqlType;
private Map<String, String> mysqlType;
private List<Map<String, String>> data;
private List<Map<String, String>> old;
private transient Message message; // 所属message
private transient CanalEntry.Entry entry; // 所属entry
public FlatMessage(long id){
this.id = id;
@@ -133,12 +133,12 @@ public class FlatMessage implements Serializable {
this.es = es;
}
public Message getMessage() {
return message;
public CanalEntry.Entry getEntry() {
return entry;
}
public void setMessage(Message message) {
this.message = message;
public void setEntry(CanalEntry.Entry entry) {
this.entry = entry;
}
/**
@@ -192,7 +192,7 @@ public class FlatMessage implements Serializable {
flatMessage.setEs(entry.getHeader().getExecuteTime());
flatMessage.setTs(System.currentTimeMillis());
flatMessage.setSql(rowChange.getSql());
flatMessage.setMessage(message);
flatMessage.setEntry(entry);
if (!rowChange.getIsDdl()) {
Map<String, Integer> sqlType = new LinkedHashMap<>();
@@ -287,7 +287,7 @@ public class FlatMessage implements Serializable {
if (flatMessage.getIsDdl()) {
partitionMessages[0] = flatMessage;
} else {
if (flatMessage.getData() != null) {
if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
String database = flatMessage.getDatabase();
String table = flatMessage.getTable();
@@ -314,7 +314,7 @@ public class FlatMessage implements Serializable {
} else {
if (pk == null) {
// 如果未指定主键(通配符主键)，从原生message中取主键字段
CanalEntry.Entry entry = flatMessage.getMessage().getEntries().get(0);
CanalEntry.Entry entry = flatMessage.getEntry();
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());

113
protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java
View File

@@ -3,7 +3,9 @@ package com.alibaba.otter.canal.protocol;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
@@ -89,4 +91,115 @@ public class Message implements Serializable {
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
}
/**
* 将 message 分区
*
* @param partitionsNum 分区数
* @param pkHashConfigs 分区库表主键正则表达式
* @return 分区message数组
*/
@SuppressWarnings("unchecked")
public Message[] messagePartition(Integer partitionsNum, String pkHashConfigs) {
if (partitionsNum == null) {
partitionsNum = 1;
}
Message[] partitionMessages = new Message[partitionsNum];
List<Entry>[] partitionEntries = new List[partitionsNum];
for (int i = 0; i < partitionsNum; i++) {
partitionEntries[i] = new ArrayList<>();
}
for (Entry entry : this.getEntries()) {
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
if (rowChange.getIsDdl()) {
partitionEntries[0].add(entry);
} else {
if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
String database = entry.getHeader().getSchemaName();
String table = entry.getHeader().getTableName();
String pk = null;
boolean isMatch = false;
String[] pkHashConfigArray = StringUtils.split(pkHashConfigs, ",");
for (String pkHashConfig : pkHashConfigArray) {
int i = pkHashConfig.lastIndexOf(".");
if (!pkHashConfig.endsWith(".$pk$")) {
// 如果指定了主键
pk = pkHashConfig.substring(i + 1);
}
pkHashConfig = pkHashConfig.substring(0, i);
isMatch = Pattern.matches(pkHashConfig, database + "." + table);
if (isMatch) {
break;
}
}
if (!isMatch) {
// 如果都没有匹配，发送到第一个分区
partitionEntries[0].add(entry);
} else {
if (pk == null) {
// 如果未指定主键(通配符主键)，取主键字段
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
CanalEntry.RowData rowData = rowChange.getRowDatasList().get(0);
for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
if (column.getIsKey()) {
pk = column.getName();
break;
}
}
}
}
if (pk == null) {
// 如果都没有匹配的主键，发送到第一个分区
partitionEntries[0].add(entry);
} else {
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
boolean hasOldPk = false;
// 如果before中有pk值说明主键有修改, 以旧的主键值hash为准
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (column.getName().equalsIgnoreCase(pk)) {
int hash = column.getValue().hashCode();
int pkHash = Math.abs(hash) % partitionsNum;
pkHash = Math.abs(pkHash);
partitionEntries[pkHash].add(entry);
hasOldPk = true;
break;
}
}
if (!hasOldPk) {
for (CanalEntry.Column column : rowData.getAfterColumnsList()) {
if (column.getName().equalsIgnoreCase(pk)) {
int hash = column.getValue().hashCode();
int pkHash = Math.abs(hash) % partitionsNum;
pkHash = Math.abs(pkHash);
partitionEntries[pkHash].add(entry);
break;
}
}
}
}
}
}
}
}
for (int i = 0; i < partitionsNum; i++) {
List<Entry> entries = partitionEntries[i];
if (!entries.isEmpty()) {
partitionMessages[i] = new Message(this.id, entries);
}
}
return partitionMessages;
}
}