chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

支持不同topic指定专属partitionNum (#2479)

Browse Source
This commit is contained in:
liguangyao
2020-08-22 13:11:53 +08:00
committed by GitHub
parent b79af46054
commit b81e9dfeb8
11 changed files with 95 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java
View File

@@ -196,6 +196,11 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
// 获取当前topic的分区数
Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, mqDestination.getDynamicTopicPartitionNum());
if (partitionNum == null) {
partitionNum = mqDestination.getPartitionsNum();
}
if (!flat) {
if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
// 并发构造
@@ -203,7 +208,7 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
// 串行分区
Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
mqDestination.getPartitionsNum(),
partitionNum,
mqDestination.getPartitionHash(),
this.mqProperties.isDatabaseHash());
int length = messages.length;
@@ -233,7 +238,7 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
for (FlatMessage flatMessage : flatMessages) {
if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
mqDestination.getPartitionsNum(),
partitionNum,
mqDestination.getPartitionHash(),
this.mqProperties.isDatabaseHash());
int length = partitionFlatMessage.length;