chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge code and fixed issue #2755 #2756 #2735 #2714 #2711 #2710 #2616 #2602 #2585 #2565 #2935 #2434 #2623 #2418 #2387

Browse Source
This commit is contained in:
agapple
2020-08-22 13:07:24 +08:00
parent 90bf9e6de5
commit b79af46054
66 changed files with 459 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java
View File

@@ -47,7 +47,6 @@ public class CanalRocketMQConsumer implements CanalMsgConsumer {
private String nameServer;
private String topic;
private String groupName;
private volatile boolean connected = false;
private DefaultMQPushConsumer rocketMQConsumer;
private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
private int batchSize = -1;
@@ -135,7 +134,6 @@ public class CanalRocketMQConsumer implements CanalMsgConsumer {
});
rocketMQConsumer.start();
} catch (MQClientException ex) {
connected = false;
logger.error("Start RocketMQ consumer error", ex);
}
}
@@ -231,6 +229,5 @@ public class CanalRocketMQConsumer implements CanalMsgConsumer {
public void disconnect() {
rocketMQConsumer.unsubscribe(topic);
rocketMQConsumer.shutdown();
connected = false;
}
}

19
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java
View File

@@ -88,6 +88,11 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
private void loadRocketMQProperties(Properties properties) {
RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
// 兼容下<=1.1.4的mq配置
doMoreCompatibleConvert("canal.mq.servers", "rocketmq.namesrv.addr", properties);
doMoreCompatibleConvert("canal.mq.producerGroup", "rocketmq.producer.group", properties);
doMoreCompatibleConvert("canal.mq.namespace", "rocketmq.namespace", properties);
doMoreCompatibleConvert("canal.mq.retries", "rocketmq.retry.times.when.send.failed", properties);
String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
if (!StringUtils.isEmpty(producerGroup)) {
@@ -121,7 +126,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
@Override
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
@@ -159,7 +164,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
if (!mqProperties.isFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
// 串行分区
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
@@ -168,7 +173,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
mqProperties.isDatabaseHash());
int length = messages.length;
ExecutorTemplate template = new ExecutorTemplate(executor);
ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
for (int i = 0; i < length; i++) {
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
if (dataPartition != null) {
@@ -190,7 +195,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
}
} else {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
// 初始化分区合并队列
@@ -211,7 +216,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
}
}
ExecutorTemplate template = new ExecutorTemplate(executor);
ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
for (int i = 0; i < partitionFlatMessages.size(); i++) {
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
if (flatMessagePart != null) {
@@ -244,7 +249,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
private void sendMessage(Message message, int partition) {
try {
SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> {
if (partition > mqs.size()) {
if (partition >= mqs.size()) {
return mqs.get(partition % mqs.size());
} else {
return mqs.get(partition);
@@ -283,7 +288,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
}
} else {
MessageQueue queue;
if (partition > size) {
if (partition >= size) {
queue = queues.get(partition % size);
} else {
queue = queues.get(partition);