chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

新增配置 canal.mq.enableDynamicQueuePartition，获取topic对应的队列的数量为分区的数量，以支持动态队列的场景（如阿里云的rocketmq，无法人工设置队列数量，可能会动态伸缩） (#3670)

Browse Source
This commit is contained in:
kkjinping
2021-10-09 15:26:23 +08:00
committed by GitHub
parent ea20076a32
commit 997957d6e0
8 changed files with 77 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java
View File

@@ -15,6 +15,7 @@ public class MQDestination {
private String partitionHash;
private String dynamicTopic;
private String dynamicTopicPartitionNum;
private Boolean enableDynamicQueuePartition;
public String getCanalDestination() {
return canalDestination;
@@ -71,4 +72,12 @@ public class MQDestination {
public void setDynamicTopicPartitionNum(String dynamicTopicPartitionNum) {
this.dynamicTopicPartitionNum = dynamicTopicPartitionNum;
}
public Boolean getEnableDynamicQueuePartition() {
return enableDynamicQueuePartition;
}
public void setEnableDynamicQueuePartition(Boolean enableDynamicQueuePartition) {
this.enableDynamicQueuePartition = enableDynamicQueuePartition;
}
}

67
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java
View File

@@ -1,29 +1,5 @@
package com.alibaba.otter.canal.connector.rocketmq.producer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException;
@@ -40,6 +16,29 @@ import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* RocketMQ Producer SPI 实现
@@ -186,6 +185,12 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
// 获取当前topic的分区数
Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
destination.getDynamicTopicPartitionNum());
// 获取topic的队列数为分区数
if(partitionNum == null){
partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
}
if (partitionNum == null) {
partitionNum = destination.getPartitionsNum();
}
@@ -353,4 +358,18 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
super.stop();
}
private Integer getTopicDynamicQueuesSize(Boolean enable, String topicName){
if(enable!=null && enable){
topicName = this.defaultMQProducer.withNamespace(topicName);
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topicName);
if(topicInfo == null){
return null;
}else{
return topicInfo.getMessageQueueList().size();
}
}
return null;
}
}