chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

支持不同topic指定专属partitionNum (#2479)

Browse Source
This commit is contained in:
liguangyao
2020-08-22 13:11:53 +08:00
committed by GitHub
parent b79af46054
commit b81e9dfeb8
11 changed files with 95 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java
View File

@@ -14,6 +14,7 @@ public class MQDestination {
private Integer partitionsNum;
private String partitionHash;
private String dynamicTopic;
private String dynamicTopicPartitionNum;
public String getCanalDestination() {
return canalDestination;
@@ -62,4 +63,12 @@ public class MQDestination {
public void setDynamicTopic(String dynamicTopic) {
this.dynamicTopic = dynamicTopic;
}
public String getDynamicTopicPartitionNum() {
return dynamicTopicPartitionNum;
}
public void setDynamicTopicPartitionNum(String dynamicTopicPartitionNum) {
this.dynamicTopicPartitionNum = dynamicTopicPartitionNum;
}
}

57
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
View File

@@ -103,6 +103,38 @@ public class MQMessageUtils {
}
});
private static Map<String, List<TopicPartitionData>> topicPartitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
.softValues(),
new Function<String, List<TopicPartitionData>>() {
public List<TopicPartitionData> apply(String tPConfigs) {
List<TopicPartitionData> datas = Lists.newArrayList();
String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs,
",",
";"),
";");
for (String tPConfig : tPArray) {
TopicPartitionData data = new TopicPartitionData();
int i = tPConfig.lastIndexOf(":");
if (i > 0) {
String tStr = tPConfig.substring(0, i);
String pStr = tPConfig.substring(i + 1);
if (!isWildCard(tStr)) {
data.simpleName = tStr;
} else {
data.regexFilter = new AviaterRegexFilter(tStr);
}
if (!StringUtils.isEmpty(pStr) && StringUtils.isNumeric(pStr)) {
data.partitionNum = Integer.valueOf(pStr);
}
datas.add(data);
}
}
return datas;
}
});
/**
* 按 schema 或者 schema+table 将 message 分配到对应topic
*
@@ -619,6 +651,24 @@ public class MQMessageUtils {
return false;
}
public static Integer parseDynamicTopicPartition(String name, String tPConfigs) {
if (!StringUtils.isEmpty(tPConfigs)) {
List<TopicPartitionData> datas = topicPartitionDatas.get(tPConfigs);
for (TopicPartitionData data : datas) {
if (data.simpleName != null) {
if (data.simpleName.equalsIgnoreCase(name)) {
return data.partitionNum;
}
} else {
if (data.regexFilter.filter(name)) {
return data.partitionNum;
}
}
}
}
return null;
}
private static boolean isWildCard(String value) {
// not contaiins '.' ?
return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
@@ -656,6 +706,13 @@ public class MQMessageUtils {
public AviaterRegexFilter tableRegexFilter;
}
public static class TopicPartitionData {
public String simpleName;
public AviaterRegexFilter regexFilter;
public Integer partitionNum;
}
public static class EntryRowData {
public Entry entry;