chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #1374 , support queryLogEvent partition

Browse Source
This commit is contained in:
agapple
2019-01-07 13:39:04 +08:00
parent 670b7830a5
commit de501c85a1
1 changed files with 40 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
View File

@@ -1,6 +1,12 @@
package com.alibaba.otter.canal.common; package com.alibaba.otter.canal.common;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -24,46 +30,36 @@ import com.google.protobuf.InvalidProtocolBufferException;
public class MQMessageUtils { public class MQMessageUtils {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<PartitionData>>() { new Function<String, List<PartitionData>>() {
public List<PartitionData> apply(String pkHashConfigs) { public List<PartitionData> apply(String pkHashConfigs) {
List<PartitionData> datas = Lists List<PartitionData> datas = Lists.newArrayList();
.newArrayList(); String[] pkHashConfigArray = StringUtils.split(pkHashConfigs,
String[] pkHashConfigArray = StringUtils ",");
.split(pkHashConfigs, ",");
// schema.table:id^name // schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) { for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData(); PartitionData data = new PartitionData();
int i = pkHashConfig int i = pkHashConfig.lastIndexOf(":");
.lastIndexOf(":");
if (i > 0) { if (i > 0) {
String pkStr = pkHashConfig String pkStr = pkHashConfig.substring(i + 1);
.substring(i + 1); if (pkStr.equalsIgnoreCase("$pk$")) {
if (pkStr.equalsIgnoreCase(
"$pk$")) {
data.hashMode.autoPkHash = true; data.hashMode.autoPkHash = true;
} else { } else {
data.hashMode.pkNames = Lists data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
.newArrayList( '^'));
StringUtils
.split(
pkStr,
'^'));
} }
pkHashConfig = pkHashConfig pkHashConfig = pkHashConfig.substring(0,
.substring(0, i); i);
} else { } else {
data.hashMode.tableHash = true; data.hashMode.tableHash = true;
} }
if (!isWildCard( if (!isWildCard(pkHashConfig)) {
pkHashConfig)) {
data.simpleName = pkHashConfig; data.simpleName = pkHashConfig;
} else { } else {
data.regexFilter = new AviaterRegexFilter( data.regexFilter = new AviaterRegexFilter(pkHashConfig);
pkHashConfig);
} }
datas.add(data); datas.add(data);
} }
@@ -73,29 +69,24 @@ public class MQMessageUtils {
}); });
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() { new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) { public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists List<DynamicTopicData> datas = Lists.newArrayList();
.newArrayList(); String[] dynamicTopicArray = StringUtils.split(pkHashConfigs,
String[] dynamicTopicArray = StringUtils ",");
.split(pkHashConfigs, ",");
// schema.table // schema.table
for (String dynamicTopic : dynamicTopicArray) { for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData(); DynamicTopicData data = new DynamicTopicData();
if (!isWildCard( if (!isWildCard(dynamicTopic)) {
dynamicTopic)) {
data.simpleName = dynamicTopic; data.simpleName = dynamicTopic;
} else { } else {
if (dynamicTopic if (dynamicTopic.contains("\\.")) {
.contains("\\.")) { data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
data.tableRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} else { } else {
data.schemaRegexFilter = new AviaterRegexFilter( data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
dynamicTopic);
} }
} }
datas.add(data); datas.add(data);
@@ -232,6 +223,9 @@ public class MQMessageUtils {
partitionEntries[pkHash].add(entry); partitionEntries[pkHash].add(entry);
} }
} }
} else {
// 针对stmt/mixed binlog格式的query事件
partitionEntries[0].add(entry);
} }
} }
} }
@@ -281,9 +275,8 @@ public class MQMessageUtils {
try { try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException( throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
"ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), + entry.toString(), e);
e);
} }
CanalEntry.EventType eventType = rowChange.getEventType(); CanalEntry.EventType eventType = rowChange.getEventType();
@@ -455,6 +448,9 @@ public class MQMessageUtils {
idx++; idx++;
} }
} }
} else {
// 针对stmt/mixed binlog格式的query事件
partitionMessages[0] = flatMessage;
} }
} }
return partitionMessages; return partitionMessages;
@@ -524,8 +520,8 @@ public class MQMessageUtils {
private static boolean isWildCard(String value) { private static boolean isWildCard(String value) {
// not contaiins '.' ? // not contaiins '.' ?
return StringUtils.containsAny(value, return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' }); '^' });
} }
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName, private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,