chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

模块化mq connector (#2562)

Browse Source 
* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
2020-02-28 11:00:08 +08:00
committed by GitHub
parent 4fc7e5f7d9
commit 265a8e8594
82 changed files with 4932 additions and 2412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
server/pom.xml
View File

@@ -25,39 +25,15 @@
<artifactId>canal.instance.manager</artifactId>
<version>${project.version}</version>
</dependency>
<!-- kafka -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>1.1.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
</dependency>
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
<dependency>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
<version>3.2.2.Final</version>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
</dependency>
<!-- test dependency -->

37
server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java
View File

@@ -1,37 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/**
* @author agapple 2019年9月29日 上午11:17:11
* @since 1.1.5
*/
public abstract class AbstractMQProducer implements CanalMQProducer {
protected ThreadPoolExecutor executor;
@Override
public void init(MQProperties mqProperties) {
int parallelThreadSize = mqProperties.getParallelThreadSize();
executor = new ThreadPoolExecutor(parallelThreadSize,
parallelThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
new NamedThreadFactory("MQParallel"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@Override
public void stop() {
executor.shutdownNow();
executor = null;
}
}

81
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java
View File

@@ -1,81 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.List;
import org.apache.kafka.common.errors.SerializationException;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
public class CanalMessageSerializer {
@SuppressWarnings("deprecation")
public static byte[] serializer(Message data, boolean filterTransactionEntry) {
try {
if (data != null) {
if (data.getId() != -1) {
if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
// for performance
List<ByteString> rowEntries = data.getRawEntries();
// message size
int messageSize = 0;
messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
int dataSize = 0;
for (int i = 0; i < rowEntries.size(); i++) {
dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
}
messageSize += dataSize;
messageSize += 1 * rowEntries.size();
// packet size
int size = 0;
size += CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
size += CodedOutputStream.computeTagSize(5)
+ CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
// build data
byte[] body = new byte[size];
CodedOutputStream output = CodedOutputStream.newInstance(body);
output.writeEnum(3, PacketType.MESSAGES.getNumber());
output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
output.writeRawVarint32(messageSize);
// message
output.writeInt64(1, data.getId());
for (int i = 0; i < rowEntries.size(); i++) {
output.writeBytes(2, rowEntries.get(i));
}
output.checkNoSpaceLeft();
return body;
} else if (!CollectionUtils.isEmpty(data.getEntries())) {
// mq模式只会走到非rowEntry模式
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) {
if (filterTransactionEntry
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
continue;
}
messageBuilder.addMessages(entry.toByteString());
}
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setVersion(NettyUtils.VERSION);
packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray();
}
}
}
} catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] ");
}
return null;
}
}

670
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java
View File

@@ -1,670 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
/**
* process MQ Message utils
*
* @author agapple 2018年12月11日 下午1:28:32
*/
/**
* @author agapple 2019年9月29日 下午12:36:26
* @since 5.0.0
*/
public class MQMessageUtils {
@SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, List<PartitionData>>() {
public List<PartitionData> apply(String pkHashConfigs) {
List<PartitionData> datas = Lists.newArrayList();
String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
// schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData();
int i = pkHashConfig.lastIndexOf(":");
if (i > 0) {
String pkStr = pkHashConfig.substring(i + 1);
if (pkStr.equalsIgnoreCase("$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
'^'));
}
pkHashConfig = pkHashConfig.substring(0,
i);
} else {
data.hashMode.tableHash = true;
}
if (!isWildCard(pkHashConfig)) {
data.simpleName = pkHashConfig;
} else {
data.regexFilter = new AviaterRegexFilter(pkHashConfig);
}
datas.add(data);
}
return datas;
}
});
@SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists.newArrayList();
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
// schema.table
for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData();
if (!isWildCard(dynamicTopic)) {
data.simpleName = dynamicTopic;
} else {
if (dynamicTopic.contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
} else {
data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
}
}
datas.add(data);
}
return datas;
}
});
/**
* 按 schema 或者 schema+table 将 message 分配到对应topic
*
* @param message 原message
* @param defaultTopic 默认topic
* @param dynamicTopicConfigs 动态topic规则
* @return 分隔后的message map
*/
public static Map<String, Message> messageTopics(Message message, String defaultTopic, String dynamicTopicConfigs) {
List<CanalEntry.Entry> entries;
if (message.isRaw()) {
List<ByteString> rawEntries = message.getRawEntries();
entries = new ArrayList<>(rawEntries.size());
for (ByteString byteString : rawEntries) {
CanalEntry.Entry entry;
try {
entry = CanalEntry.Entry.parseFrom(byteString);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
entries.add(entry);
}
} else {
entries = message.getEntries();
}
Map<String, Message> messages = new HashMap<>();
for (CanalEntry.Entry entry : entries) {
// 如果有topic路由,则忽略begin/end事件
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
String schemaName = entry.getHeader().getSchemaName();
String tableName = entry.getHeader().getTableName();
if (StringUtils.isEmpty(schemaName) || StringUtils.isEmpty(tableName)) {
put2MapMessage(messages, message.getId(), defaultTopic, entry);
} else {
Set<String> topics = matchTopics(schemaName + "." + tableName, dynamicTopicConfigs);
if (topics != null) {
for (String topic : topics) {
put2MapMessage(messages, message.getId(), topic, entry);
}
} else {
topics = matchTopics(schemaName, dynamicTopicConfigs);
if (topics != null) {
for (String topic : topics) {
put2MapMessage(messages, message.getId(), topic, entry);
}
} else {
put2MapMessage(messages, message.getId(), defaultTopic, entry);
}
}
}
}
return messages;
}
/**
* 多线程构造message的rowChanged对象，比如为partition/flastMessage转化等处理 </br>
* 因为protobuf对象的序列化和反序列化是cpu密集型，串行执行会有代价
*/
public static EntryRowData[] buildMessageData(Message message, ThreadPoolExecutor executor) {
ExecutorTemplate template = new ExecutorTemplate(executor);
if (message.isRaw()) {
List<ByteString> rawEntries = message.getRawEntries();
final EntryRowData[] datas = new EntryRowData[rawEntries.size()];
int i = 0;
for (ByteString byteString : rawEntries) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
try {
Entry entry = Entry.parseFrom(byteString);
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
datas[index] = new EntryRowData();
datas[index].entry = entry;
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
}
});
i++;
}
template.waitForResult();
return datas;
} else {
final EntryRowData[] datas = new EntryRowData[message.getEntries().size()];
int i = 0;
for (Entry entry : message.getEntries()) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
datas[index] = new EntryRowData();
datas[index].entry = entry;
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
}
});
i++;
}
template.waitForResult();
return datas;
}
}
/**
* 将 message 分区
*
* @param partitionsNum 分区数
* @param pkHashConfigs 分区库表主键正则表达式
* @param databaseHash 是否取消根据database进行hash
* @return 分区message数组
*/
@SuppressWarnings("unchecked")
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum,
String pkHashConfigs, boolean databaseHash) {
if (partitionsNum == null) {
partitionsNum = 1;
}
Message[] partitionMessages = new Message[partitionsNum];
List<Entry>[] partitionEntries = new List[partitionsNum];
for (int i = 0; i < partitionsNum; i++) {
// 注意一下并发
partitionEntries[i] = Collections.synchronizedList(Lists.newArrayList());
}
for (EntryRowData data : datas) {
CanalEntry.Entry entry = data.entry;
CanalEntry.RowChange rowChange = data.rowChange;
// 如果有分区路由,则忽略begin/end事件
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
if (rowChange.getIsDdl()) {
partitionEntries[0].add(entry);
} else {
if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
String database = entry.getHeader().getSchemaName();
String table = entry.getHeader().getTableName();
HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
if (hashMode == null) {
// 如果都没有匹配，发送到第一个分区
partitionEntries[0].add(entry);
} else if (hashMode.tableHash) {
int hashCode = table.hashCode();
int pkHash = Math.abs(hashCode) % partitionsNum;
pkHash = Math.abs(pkHash);
// tableHash not need split entry message
partitionEntries[pkHash].add(entry);
} else {
// build new entry
Entry.Builder builder = Entry.newBuilder(entry);
RowChange.Builder rowChangeBuilder = RowChange.newBuilder(rowChange);
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
int hashCode = 0;
if (databaseHash) {
hashCode = database.hashCode();
}
CanalEntry.EventType eventType = rowChange.getEventType();
List<CanalEntry.Column> columns = null;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
if (hashMode.autoPkHash) {
// isEmpty use default pkNames
for (CanalEntry.Column column : columns) {
if (column.getIsKey()) {
hashCode = hashCode ^ column.getValue().hashCode();
}
}
} else {
for (CanalEntry.Column column : columns) {
if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
hashCode = hashCode ^ column.getValue().hashCode();
}
}
}
int pkHash = Math.abs(hashCode) % partitionsNum;
pkHash = Math.abs(pkHash);
// clear rowDatas
rowChangeBuilder.clearRowDatas();
rowChangeBuilder.addRowDatas(rowData);
builder.clearStoreValue();
builder.setStoreValue(rowChangeBuilder.build().toByteString());
partitionEntries[pkHash].add(builder.build());
}
}
} else {
// 针对stmt/mixed binlog格式的query事件
partitionEntries[0].add(entry);
}
}
}
for (int i = 0; i < partitionsNum; i++) {
List<Entry> entriesTmp = partitionEntries[i];
if (!entriesTmp.isEmpty()) {
partitionMessages[i] = new Message(id, entriesTmp);
}
}
return partitionMessages;
}
/**
* 将Message转换为FlatMessage
*
* @param message 原生message
* @return FlatMessage列表
*/
public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
List<FlatMessage> flatMessages = new ArrayList<>();
for (EntryRowData entryRowData : datas) {
CanalEntry.Entry entry = entryRowData.entry;
CanalEntry.RowChange rowChange = entryRowData.rowChange;
// 如果有分区路由,则忽略begin/end事件
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
// build flatMessage
CanalEntry.EventType eventType = rowChange.getEventType();
FlatMessage flatMessage = new FlatMessage(id);
flatMessages.add(flatMessage);
flatMessage.setDatabase(entry.getHeader().getSchemaName());
flatMessage.setTable(entry.getHeader().getTableName());
flatMessage.setIsDdl(rowChange.getIsDdl());
flatMessage.setType(eventType.toString());
flatMessage.setEs(entry.getHeader().getExecuteTime());
flatMessage.setTs(System.currentTimeMillis());
flatMessage.setSql(rowChange.getSql());
if (!rowChange.getIsDdl()) {
Map<String, Integer> sqlType = new LinkedHashMap<>();
Map<String, String> mysqlType = new LinkedHashMap<>();
List<Map<String, String>> data = new ArrayList<>();
List<Map<String, String>> old = new ArrayList<>();
Set<String> updateSet = new HashSet<>();
boolean hasInitPkNames = false;
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) {
continue;
}
Map<String, String> row = new LinkedHashMap<>();
List<CanalEntry.Column> columns;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
for (CanalEntry.Column column : columns) {
if (!hasInitPkNames && column.getIsKey()) {
flatMessage.addPkName(column.getName());
}
sqlType.put(column.getName(), column.getSqlType());
mysqlType.put(column.getName(), column.getMysqlType());
if (column.getIsNull()) {
row.put(column.getName(), null);
} else {
row.put(column.getName(), column.getValue());
}
// 获取update为true的字段
if (column.getUpdated()) {
updateSet.add(column.getName());
}
}
hasInitPkNames = true;
if (!row.isEmpty()) {
data.add(row);
}
if (eventType == CanalEntry.EventType.UPDATE) {
Map<String, String> rowOld = new LinkedHashMap<>();
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
if (column.getIsNull()) {
rowOld.put(column.getName(), null);
} else {
rowOld.put(column.getName(), column.getValue());
}
}
}
// update操作将记录修改前的值
if (!rowOld.isEmpty()) {
old.add(rowOld);
}
}
}
if (!sqlType.isEmpty()) {
flatMessage.setSqlType(sqlType);
}
if (!mysqlType.isEmpty()) {
flatMessage.setMysqlType(mysqlType);
}
if (!data.isEmpty()) {
flatMessage.setData(data);
}
if (!old.isEmpty()) {
flatMessage.setOld(old);
}
}
}
return flatMessages;
}
/**
* 将FlatMessage按指定的字段值hash拆分
*
* @param flatMessage flatMessage
* @param partitionsNum 分区数量
* @param pkHashConfigs hash映射
* @param databaseHash 是否取消根据database进行hash
* @return 拆分后的flatMessage数组
*/
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum, String pkHashConfigs,
boolean databaseHash) {
if (partitionsNum == null) {
partitionsNum = 1;
}
FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
if (flatMessage.getIsDdl()) {
partitionMessages[0] = flatMessage;
} else {
if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
String database = flatMessage.getDatabase();
String table = flatMessage.getTable();
HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
if (hashMode == null) {
// 如果都没有匹配，发送到第一个分区
partitionMessages[0] = flatMessage;
} else if (hashMode.tableHash) {
int hashCode = table.hashCode();
int pkHash = Math.abs(hashCode) % partitionsNum;
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
pkHash = Math.abs(pkHash);
partitionMessages[pkHash] = flatMessage;
} else {
List<String> pkNames = hashMode.pkNames;
if (hashMode.autoPkHash) {
pkNames = flatMessage.getPkNames();
}
int idx = 0;
for (Map<String, String> row : flatMessage.getData()) {
int hashCode = 0;
if (databaseHash) {
hashCode = database.hashCode();
}
if (pkNames != null) {
for (String pkName : pkNames) {
String value = row.get(pkName);
if (value == null) {
value = "";
}
hashCode = hashCode ^ value.hashCode();
}
}
int pkHash = Math.abs(hashCode) % partitionsNum;
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
pkHash = Math.abs(pkHash);
FlatMessage flatMessageTmp = partitionMessages[pkHash];
if (flatMessageTmp == null) {
flatMessageTmp = new FlatMessage(flatMessage.getId());
partitionMessages[pkHash] = flatMessageTmp;
flatMessageTmp.setDatabase(flatMessage.getDatabase());
flatMessageTmp.setTable(flatMessage.getTable());
flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
flatMessageTmp.setType(flatMessage.getType());
flatMessageTmp.setSql(flatMessage.getSql());
flatMessageTmp.setSqlType(flatMessage.getSqlType());
flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
flatMessageTmp.setEs(flatMessage.getEs());
flatMessageTmp.setTs(flatMessage.getTs());
flatMessageTmp.setPkNames(flatMessage.getPkNames());
}
List<Map<String, String>> data = flatMessageTmp.getData();
if (data == null) {
data = new ArrayList<>();
flatMessageTmp.setData(data);
}
data.add(row);
if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
List<Map<String, String>> old = flatMessageTmp.getOld();
if (old == null) {
old = new ArrayList<>();
flatMessageTmp.setOld(old);
}
old.add(flatMessage.getOld().get(idx));
}
idx++;
}
}
} else {
// 针对stmt/mixed binlog格式的query事件
partitionMessages[0] = flatMessage;
}
}
return partitionMessages;
}
/**
* match return List , not match return null
*/
public static HashMode getPartitionHashColumns(String name, String pkHashConfigs) {
if (StringUtils.isEmpty(pkHashConfigs)) {
return null;
}
List<PartitionData> datas = partitionDatas.get(pkHashConfigs);
for (PartitionData data : datas) {
if (data.simpleName != null) {
if (data.simpleName.equalsIgnoreCase(name)) {
return data.hashMode;
}
} else {
if (data.regexFilter.filter(name)) {
return data.hashMode;
}
}
}
return null;
}
private static Set<String> matchTopics(String name, String dynamicTopicConfigs) {
String[] router = StringUtils.split(StringUtils.replace(dynamicTopicConfigs, ",", ";"), ";");
Set<String> topics = new HashSet<>();
for (String item : router) {
int i = item.indexOf(":");
if (i > -1) {
String topic = item.substring(0, i).trim();
String topicConfigs = item.substring(i + 1).trim();
if (matchDynamicTopic(name, topicConfigs)) {
topics.add(topic);
// 匹配了一个就退出
break;
}
} else if (matchDynamicTopic(name, item)) {
// 匹配了一个就退出
topics.add(name.toLowerCase());
break;
}
}
return topics.isEmpty() ? null : topics;
}
public static boolean matchDynamicTopic(String name, String dynamicTopicConfigs) {
if (StringUtils.isEmpty(dynamicTopicConfigs)) {
return false;
}
boolean res = false;
List<DynamicTopicData> datas = dynamicTopicDatas.get(dynamicTopicConfigs);
for (DynamicTopicData data : datas) {
if (data.simpleName != null) {
if (data.simpleName.equalsIgnoreCase(name)) {
res = true;
break;
}
} else if (name.contains(".")) {
if (data.tableRegexFilter != null && data.tableRegexFilter.filter(name)) {
res = true;
break;
}
} else {
if (data.schemaRegexFilter != null && data.schemaRegexFilter.filter(name)) {
res = true;
break;
}
}
}
return res;
}
public static boolean checkPkNamesHasContain(List<String> pkNames, String name) {
for (String pkName : pkNames) {
if (pkName.equalsIgnoreCase(name)) {
return true;
}
}
return false;
}
private static boolean isWildCard(String value) {
// not contaiins '.' ?
return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
'^' });
}
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,
CanalEntry.Entry entry) {
Message message = messageMap.get(topicName);
if (message == null) {
message = new Message(messageId, new ArrayList<CanalEntry.Entry>());
messageMap.put(topicName, message);
}
message.getEntries().add(entry);
}
public static class PartitionData {
public String simpleName;
public AviaterRegexFilter regexFilter;
public HashMode hashMode = new HashMode();
}
public static class HashMode {
public boolean autoPkHash = false;
public boolean tableHash = false;
public List<String> pkNames = Lists.newArrayList();
}
public static class DynamicTopicData {
public String simpleName;
public AviaterRegexFilter schemaRegexFilter;
public AviaterRegexFilter tableRegexFilter;
}
public static class EntryRowData {
public Entry entry;
public RowChange rowChange;
}
}

368
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -1,368 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.Properties;
/**
* MQ 配置项
*
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class MQProperties {
private String servers = "127.0.0.1:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private int maxRequestSize = 1048576;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout = 100L;
private boolean flatMessage = true;
private String compressionType = "none";
private String acks = "all";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private Properties properties = new Properties();
private boolean enableMessageTrace = false;
private String accessChannel = null;
private String customizedTraceTopic = null;
private String namespace = "";
// kafka集群是否启动Kerberos认证
private boolean kerberosEnable = false;
// 启动Kerberos认证时配置为krb5.conf文件的路径
private String kerberosKrb5FilePath = "";
// 启动Kerberos认证时配置为jaas.conf文件的路径
private String kerberosJaasFilePath = "";
// rabbitmq 账号
private String username = "";
// rabbitmq 密码
private String password = "";
// rabbitmq 密码
private String vhost = "";
// aliyun 用户ID rabbitmq 阿里云需要使用
private long aliyunUID = 0;
// rabbitmq 交换机
private String exchange = "";
// 消息发送的并行度
private int parallelThreadSize = 8;
// 是否取消根据database进行hash
private boolean databaseHash = true;
public static class CanalDestination {
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private String partitionHash;
private String dynamicTopic;
public String getCanalDestination() {
return canalDestination;
}
public void setCanalDestination(String canalDestination) {
this.canalDestination = canalDestination;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Integer getPartition() {
return partition;
}
public void setPartition(Integer partition) {
this.partition = partition;
}
public Integer getPartitionsNum() {
return partitionsNum;
}
public void setPartitionsNum(Integer partitionsNum) {
this.partitionsNum = partitionsNum;
}
public String getPartitionHash() {
return partitionHash;
}
public void setPartitionHash(String partitionHash) {
this.partitionHash = partitionHash;
}
public String getDynamicTopic() {
return dynamicTopic;
}
public void setDynamicTopic(String dynamicTopic) {
this.dynamicTopic = dynamicTopic;
}
}
public String getServers() {
return servers;
}
public void setServers(String servers) {
this.servers = servers;
}
public int getRetries() {
return retries;
}
public void setRetries(int retries) {
this.retries = retries;
}
public int getBatchSize() {
return batchSize;
}
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
public int getLingerMs() {
return lingerMs;
}
public void setLingerMs(int lingerMs) {
this.lingerMs = lingerMs;
}
public long getBufferMemory() {
return bufferMemory;
}
public void setBufferMemory(long bufferMemory) {
this.bufferMemory = bufferMemory;
}
public int getCanalBatchSize() {
return canalBatchSize;
}
public void setCanalBatchSize(int canalBatchSize) {
this.canalBatchSize = canalBatchSize;
}
public Long getCanalGetTimeout() {
return canalGetTimeout;
}
public void setCanalGetTimeout(Long canalGetTimeout) {
this.canalGetTimeout = canalGetTimeout;
}
public boolean getFlatMessage() {
return flatMessage;
}
public void setFlatMessage(boolean flatMessage) {
this.flatMessage = flatMessage;
}
public boolean isFilterTransactionEntry() {
return filterTransactionEntry;
}
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
this.filterTransactionEntry = filterTransactionEntry;
}
public String getProducerGroup() {
return producerGroup;
}
public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup;
}
public String getCompressionType() {
return compressionType;
}
public void setCompressionType(String compressionType) {
this.compressionType = compressionType;
}
public String getAcks() {
return acks;
}
public void setAcks(String acks) {
this.acks = acks;
}
public String getAliyunAccessKey() {
return aliyunAccessKey;
}
public void setAliyunAccessKey(String aliyunAccessKey) {
this.aliyunAccessKey = aliyunAccessKey;
}
public String getAliyunSecretKey() {
return aliyunSecretKey;
}
public void setAliyunSecretKey(String aliyunSecretKey) {
this.aliyunSecretKey = aliyunSecretKey;
}
public int getMaxRequestSize() {
return maxRequestSize;
}
public void setMaxRequestSize(int maxRequestSize) {
this.maxRequestSize = maxRequestSize;
}
public Properties getProperties() {
return properties;
}
public void setProperties(Properties properties) {
this.properties = properties;
}
public boolean isEnableMessageTrace() {
return enableMessageTrace;
}
public void setEnableMessageTrace(boolean enableMessageTrace) {
this.enableMessageTrace = enableMessageTrace;
}
public String getAccessChannel() {
return accessChannel;
}
public void setAccessChannel(String accessChannel) {
this.accessChannel = accessChannel;
}
public String getCustomizedTraceTopic() {
return customizedTraceTopic;
}
public void setCustomizedTraceTopic(String customizedTraceTopic) {
this.customizedTraceTopic = customizedTraceTopic;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
public boolean isKerberosEnable() {
return kerberosEnable;
}
public void setKerberosEnable(boolean kerberosEnable) {
this.kerberosEnable = kerberosEnable;
}
public String getKerberosKrb5FilePath() {
return kerberosKrb5FilePath;
}
public void setKerberosKrb5FilePath(String kerberosKrb5FilePath) {
this.kerberosKrb5FilePath = kerberosKrb5FilePath;
}
public String getKerberosJaasFilePath() {
return kerberosJaasFilePath;
}
public void setKerberosJaasFilePath(String kerberosJaasFilePath) {
this.kerberosJaasFilePath = kerberosJaasFilePath;
}
public void setPassword(String password) {
this.password = password;
}
public String getPassword() {
return password;
}
public void setUsername(String username) {
this.username = username;
}
public String getUsername() {
return username;
}
public void setVhost(String vhost) {
this.vhost = vhost;
}
public String getVhost() {
return vhost;
}
public long getAliyunUID() {
return aliyunUID;
}
public void setAliyunUID(long aliyunUID) {
this.aliyunUID = aliyunUID;
}
public String getExchange() {
return exchange;
}
public void setExchange(String exchange) {
this.exchange = exchange;
}
public int getParallelThreadSize() {
return parallelThreadSize;
}
public void setParallelThreadSize(int parallelThreadSize) {
this.parallelThreadSize = parallelThreadSize;
}
public boolean getDatabaseHash() {
return databaseHash;
}
public void setDatabaseHash(boolean databaseHash) {
this.databaseHash = databaseHash;
}
@Override
public String toString() {
return "MQProperties [servers=" + servers + ", retries=" + retries + ", batchSize=" + batchSize + ", lingerMs="
+ lingerMs + ", maxRequestSize=" + maxRequestSize + ", bufferMemory=" + bufferMemory
+ ", filterTransactionEntry=" + filterTransactionEntry + ", producerGroup=" + producerGroup
+ ", canalBatchSize=" + canalBatchSize + ", canalGetTimeout=" + canalGetTimeout + ", flatMessage="
+ flatMessage + ", compressionType=" + compressionType + ", acks=" + acks + ", aliyunAccessKey="
+ aliyunAccessKey + ", aliyunSecretKey=" + aliyunSecretKey + ", properties=" + properties
+ ", enableMessageTrace=" + enableMessageTrace + ", accessChannel=" + accessChannel
+ ", customizedTraceTopic=" + customizedTraceTopic + ", namespace=" + namespace + ", kerberosEnable="
+ kerberosEnable + ", kerberosKrb5FilePath=" + kerberosKrb5FilePath + ", kerberosJaasFilePath="
+ kerberosJaasFilePath + ", username=" + username + ", password=" + password + ", vhost=" + vhost
+ ", aliyunUID=" + aliyunUID + ", exchange=" + exchange + ", parallelThreadSize=" + parallelThreadSize
+ ",databaseHash=" + databaseHash + "]";
}
}

240
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,240 +0,0 @@
package com.alibaba.otter.canal.kafka;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.commons.lang.StringUtils;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/**
* kafka producer 主操作类
*
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private Producer<String, byte[]> producer;
private MQProperties kafkaProperties;
@Override
public void init(MQProperties kafkaProperties) {
super.init(kafkaProperties);
this.kafkaProperties = kafkaProperties;
Properties properties = new Properties();
properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", kafkaProperties.getAcks());
properties.put("compression.type", kafkaProperties.getCompressionType());
properties.put("batch.size", kafkaProperties.getBatchSize());
properties.put("linger.ms", kafkaProperties.getLingerMs());
properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
properties.put("buffer.memory", kafkaProperties.getBufferMemory());
properties.put("key.serializer", StringSerializer.class.getName());
properties.put("max.in.flight.requests.per.connection", 1);
if (!kafkaProperties.getProperties().isEmpty()) {
properties.putAll(kafkaProperties.getProperties());
}
properties.put("retries", kafkaProperties.getRetries());
if (kafkaProperties.isKerberosEnable()) {
File krb5File = new File(kafkaProperties.getKerberosKrb5FilePath());
File jaasFile = new File(kafkaProperties.getKerberosJaasFilePath());
if (krb5File.exists() && jaasFile.exists()) {
// 配置kerberos认证，需要使用绝对路径
System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath());
System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath());
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
properties.put("security.protocol", "SASL_PLAINTEXT");
properties.put("sasl.kerberos.service.name", "kafka");
} else {
String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it";
logger.error(errorMsg);
throw new RuntimeException(errorMsg);
}
}
properties.put("value.serializer", KafkaMessageSerializer.class.getName());
producer = new KafkaProducer<String, byte[]>(properties);
}
@Override
public void stop() {
try {
logger.info("## stop the kafka producer");
if (producer != null) {
producer.close();
}
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka producer:", e);
} finally {
logger.info("## kafka producer is down.");
}
super.stop();
}
@Override
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
boolean flat = kafkaProperties.getFlatMessage();
try {
List result = null;
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
// 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
Map<String, Message> messageMap = MQMessageUtils.messageTopics(message,
canalDestination.getTopic(),
canalDestination.getDynamicTopic());
// 针对不同的topic,引入多线程提升效率
for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final Message messageSub = entry.getValue();
template.submit(new Callable() {
@Override
public List<Future> call() throws Exception {
try {
return send(canalDestination, topicName, messageSub, flat);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
result = template.waitForResult();
} else {
result = new ArrayList();
List<Future> futures = send(canalDestination, canalDestination.getTopic(), message, flat);
result.add(futures);
}
// 一个批次的所有topic和分区的队列，都采用异步的模式进行多线程批量发送
// 最后在集结点进行flush等待，确保所有数据都写出成功
// 注意：kafka的异步模式如果要保证顺序性，需要设置max.in.flight.requests.per.connection=1，确保在网络异常重试时有排他性
producer.flush();
// flush操作也有可能是发送失败,这里需要异步关注一下发送结果,针对有异常的直接出发rollback
for (Object obj : result) {
List<Future> futures = (List<Future>) obj;
for (Future future : futures) {
try {
future.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
private List<Future> send(MQProperties.CanalDestination canalDestination, String topicName, Message message,
boolean flat) throws Exception {
List<ProducerRecord<String, byte[]>> records = new ArrayList<ProducerRecord<String, byte[]>>();
if (!flat) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
canalDestination.getPartitionsNum(),
canalDestination.getPartitionHash(),
kafkaProperties.getDatabaseHash());
int length = messages.length;
for (int i = 0; i < length; i++) {
Message messagePartition = messages[i];
if (messagePartition != null) {
records.add(new ProducerRecord<String, byte[]>(topicName,
i,
null,
CanalMessageSerializer.serializer(messagePartition,
kafkaProperties.isFilterTransactionEntry())));
}
}
} else {
final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0;
records.add(new ProducerRecord<String, byte[]>(topicName,
partition,
null,
CanalMessageSerializer.serializer(message, kafkaProperties.isFilterTransactionEntry())));
}
} else {
// 发送扁平数据json
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
canalDestination.getPartitionsNum(),
canalDestination.getPartitionHash(),
kafkaProperties.getDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) {
records.add(new ProducerRecord<String, byte[]>(topicName,
i,
null,
JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
}
}
} else {
final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0;
records.add(new ProducerRecord<String, byte[]>(topicName,
partition,
null,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
}
}
}
return produce(topicName, records, flat);
}
private List<Future> produce(String topicName, List<ProducerRecord<String, byte[]>> records, boolean flatMessage) {
List<Future> futures = new ArrayList<Future>();
// 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题
for (ProducerRecord record : records) {
futures.add(producer.send(record));
}
return futures;
}
}

28
server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java
View File

@@ -1,28 +0,0 @@
package com.alibaba.otter.canal.kafka;
import java.util.Map;
import org.apache.kafka.common.serialization.Serializer;
/**
* Kafka Message类的序列化
*
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class KafkaMessageSerializer implements Serializer<byte[]> {
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
}
@Override
public byte[] serialize(String topic, byte[] data) {
return data;
}
@Override
public void close() {
// nothing to do
}
}

50
server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java
View File

@@ -1,50 +0,0 @@
/**
* aliyun amqp协议 账号类
* 暂不支持STS授权情况
*/
package com.alibaba.otter.canal.rabbitmq;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import com.alibaba.mq.amqp.utils.UserUtils;
import com.rabbitmq.client.impl.CredentialsProvider;
public class AliyunCredentialsProvider implements CredentialsProvider {
/**
* Access Key ID
*/
private final String AliyunAccessKey;
/**
* Access Key Secret
*/
private final String AliyunAccessSecret;
/**
* 资源主账号ID
*/
private final long resourceOwnerId;
public AliyunCredentialsProvider(final String accessKey, final String accessSecret, final long resourceOwnerId){
this.AliyunAccessKey = accessKey;
this.AliyunAccessSecret = accessSecret;
this.resourceOwnerId = resourceOwnerId;
}
@Override
public String getUsername() {
return UserUtils.getUserName(AliyunAccessKey, resourceOwnerId);
}
@Override
public String getPassword() {
try {
return UserUtils.getPassord(AliyunAccessSecret);
} catch (InvalidKeyException | NoSuchAlgorithmException ignored) {
}
return null;
}
}

145
server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java
View File

@@ -1,145 +0,0 @@
package com.alibaba.otter.canal.rabbitmq;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
private MQProperties mqProperties;
private Connection connect;
private Channel channel;
@Override
public void init(MQProperties mqProperties) {
super.init(mqProperties);
this.mqProperties = mqProperties;
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(mqProperties.getServers());
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
&& mqProperties.getAliyunUID() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
mqProperties.getAliyunSecretKey(),
mqProperties.getAliyunUID()));
} else {
factory.setUsername(mqProperties.getUsername());
factory.setPassword(mqProperties.getPassword());
}
factory.setVirtualHost(mqProperties.getVhost());
try {
connect = factory.newConnection();
channel = connect.createChannel();
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Start RabbitMQ producer error", ex);
}
}
@Override
public void send(final MQProperties.CanalDestination canalDestination, Message message, Callback callback)
throws IOException {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
canalDestination.getTopic(),
canalDestination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
send(canalDestination, topicName, messageSub);
}
});
}
template.waitForResult();
} else {
send(canalDestination, canalDestination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
private void send(MQProperties.CanalDestination canalDestination, String topicName, Message messageSub) {
if (!mqProperties.getFlatMessage()) {
byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry());
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
} else {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}",
message,
canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
}
}
}
}
private void sendMessage(String queueName, byte[] message) {
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
try {
channel.basicPublish(mqProperties.getExchange(), queueName, null, message);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void stop() {
logger.info("## Stop RabbitMQ producer##");
try {
this.connect.close();
this.channel.close();
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Stop RabbitMQ producer error", ex);
}
super.stop();
}
}

285
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
View File

@@ -1,285 +0,0 @@
package com.alibaba.otter.canal.rocketmq;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
private MQProperties mqProperties;
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
@Override
public void init(MQProperties rocketMQProperties) {
super.init(rocketMQProperties);
this.mqProperties = rocketMQProperties;
RPCHook rpcHook = null;
if (rocketMQProperties.getAliyunAccessKey().length() > 0
&& rocketMQProperties.getAliyunSecretKey().length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
rpcHook = new AclClientRPCHook(sessionCredentials);
}
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
rpcHook,
mqProperties.isEnableMessageTrace(),
mqProperties.getCustomizedTraceTopic());
if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(mqProperties.getNamespace())) {
defaultMQProducer.setNamespace(mqProperties.getNamespace());
}
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
defaultMQProducer.setVipChannelEnabled(false);
logger.info("##Start RocketMQ producer##");
try {
defaultMQProducer.start();
} catch (MQClientException ex) {
throw new CanalServerException("Start RocketMQ producer error", ex);
}
}
@Override
public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(data,
destination.getTopic(),
destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
try {
send(destination, topicName, messageSub);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), data);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
public void send(final MQProperties.CanalDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) throws Exception {
if (!mqProperties.getFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.getDatabaseHash());
int length = messages.length;
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < length; i++) {
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
if (dataPartition != null) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
Message data = new Message(topicName, CanalMessageSerializer.serializer(dataPartition,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, index);
}
});
}
}
// 等所有分片发送完毕
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
Message data = new Message(topicName, CanalMessageSerializer.serializer(message,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, partition);
}
} else {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
if (flatMessages != null) {
// 初始化分区合并队列
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
List<List<FlatMessage>> partitionFlatMessages = new ArrayList<List<FlatMessage>>();
for (int i = 0; i < destination.getPartitionsNum(); i++) {
partitionFlatMessages.add(new ArrayList<FlatMessage>());
}
for (FlatMessage flatMessage : flatMessages) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.getDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
}
}
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < partitionFlatMessages.size(); i++) {
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
if (flatMessagePart != null) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
List<Message> messages = flatMessagePart.stream()
.map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, index);
}
});
}
}
// 批量等所有分区的结果
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
List<Message> messages = flatMessages.stream()
.map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, partition);
}
}
}
}
private void sendMessage(Message message, int partition) {
try {
SendResult sendResult = this.defaultMQProducer.send(message, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
if (partition > mqs.size()) {
return mqs.get(partition % mqs.size());
} else {
return mqs.get(partition);
}
}
}, null);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("deprecation")
private void sendMessage(List<Message> messages, int partition) {
if (messages.isEmpty()) {
return;
}
// 获取一下messageQueue
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
if (topicInfo == null) {
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
// 批量发送
List<MessageQueue> queues = topicInfo.getMessageQueueList();
int size = queues.size();
if (size <= 0) {
// 可能是第一次创建
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
MessageQueue queue = null;
if (partition > size) {
queue = queues.get(partition % size);
} else {
queue = queues.get(partition);
}
try {
SendResult sendResult = this.defaultMQProducer.send(messages, queue);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
}
}
@Override
public void stop() {
logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown();
super.stop();
}
}

30
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -7,18 +7,21 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalMQConfig;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
public class CanalMQStarter {
@@ -30,7 +33,7 @@ public class CanalMQStarter {
private CanalMQProducer canalMQProducer;
private MQProperties properties;
private MQProperties mqProperties;
private CanalServerWithEmbedded canalServer;
@@ -42,15 +45,14 @@ public class CanalMQStarter {
this.canalMQProducer = canalMQProducer;
}
public synchronized void start(MQProperties properties, String destinations) {
public synchronized void start(String destinations) {
try {
if (running) {
return;
}
this.properties = properties;
canalMQProducer.init(properties);
mqProperties = canalMQProducer.getMqProperties();
// set filterTransactionEntry
if (properties.isFilterTransactionEntry()) {
if (mqProperties.isFilterTransactionEntry()) {
System.setProperty("canal.instance.filter.transaction.entry", "true");
}
@@ -151,7 +153,7 @@ public class CanalMQStarter {
}
continue;
}
MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination();
MQDestination canalDestination = new MQDestination();
canalDestination.setCanalDestination(destination);
CanalMQConfig mqConfig = canalInstance.getMqConfig();
canalDestination.setTopic(mqConfig.getTopic());
@@ -163,15 +165,13 @@ public class CanalMQStarter {
canalServer.subscribe(clientIdentity);
logger.info("## the MQ producer: {} is running now ......", destination);
Long getTimeout = properties.getCanalGetTimeout();
int getBatchSize = properties.getCanalBatchSize();
Integer getTimeout = mqProperties.getFetchTimeout();
Integer getBatchSize = mqProperties.getBatchSize();
while (running && destinationRunning.get()) {
Message message;
if (getTimeout != null && getTimeout > 0) {
message = canalServer.getWithoutAck(clientIdentity,
getBatchSize,
getTimeout,
TimeUnit.MILLISECONDS);
message = canalServer
.getWithoutAck(clientIdentity, getBatchSize, getTimeout.longValue(), TimeUnit.MILLISECONDS);
} else {
message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
}
@@ -180,7 +180,7 @@ public class CanalMQStarter {
try {
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) {
canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() {
canalMQProducer.send(canalDestination, message, new Callback() {
@Override
public void commit() {

37
server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java
View File

@@ -1,37 +0,0 @@
package com.alibaba.otter.canal.spi;
import java.io.IOException;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.Message;
public interface CanalMQProducer {
/**
* Init producer.
*
* @param mqProperties MQ config
*/
void init(MQProperties mqProperties);
/**
* Send canal message to related topic
*
* @param canalDestination canal mq destination
* @param message canal message
* @throws IOException
*/
void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException;
/**
* Stop MQ producer service
*/
void stop();
interface Callback {
void commit();
void rollback();
}
}