mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
对原生message发送mq的分区支持
This commit is contained in:
machey
@@ -80,22 +80,36 @@ public class CanalKafkaProducer implements CanalMQProducer {
|
||||
// producer.beginTransaction();
|
||||
if (!kafkaProperties.getFlatMessage()) {
|
||||
try {
|
||||
ProducerRecord<String, Message> record;
|
||||
ProducerRecord<String, Message> record = null;
|
||||
if (canalDestination.getPartition() != null) {
|
||||
record = new ProducerRecord<String, Message>(canalDestination.getTopic(),
|
||||
record = new ProducerRecord<>(canalDestination.getTopic(),
|
||||
canalDestination.getPartition(),
|
||||
null,
|
||||
message);
|
||||
} else {
|
||||
record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message);
|
||||
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
|
||||
Message[] messages = message.messagePartition(canalDestination.getPartitionsNum(),
|
||||
canalDestination.getPartitionHash());
|
||||
int length = messages.length;
|
||||
for (int i = 0; i < length; i++) {
|
||||
Message messagePartition = messages[i];
|
||||
if (messagePartition != null) {
|
||||
record = new ProducerRecord<>(canalDestination.getTopic(), i, null, messagePartition);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
record = new ProducerRecord<>(canalDestination.getTopic(), 0, null, message);
|
||||
}
|
||||
}
|
||||
|
||||
producer.send(record).get();
|
||||
if (record != null) {
|
||||
producer.send(record).get();
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Send message to kafka topic: [{}], packet: {}",
|
||||
canalDestination.getTopic(),
|
||||
message.toString());
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Send message to kafka topic: [{}], packet: {}",
|
||||
canalDestination.getTopic(),
|
||||
message.toString());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
|
||||
@@ -1,14 +1,7 @@
|
||||
package com.alibaba.otter.canal.rocketmq;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import com.alibaba.otter.canal.common.CanalMessageSerializer;
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.server.exception.CanalServerException;
|
||||
import com.alibaba.otter.canal.spi.CanalMQProducer;
|
||||
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
|
||||
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.rocketmq.client.exception.MQBrokerException;
|
||||
import org.apache.rocketmq.client.exception.MQClientException;
|
||||
import org.apache.rocketmq.client.producer.DefaultMQProducer;
|
||||
@@ -20,7 +13,15 @@ import org.apache.rocketmq.remoting.exception.RemotingException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.List;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import com.alibaba.otter.canal.common.CanalMessageSerializer;
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.server.exception.CanalServerException;
|
||||
import com.alibaba.otter.canal.spi.CanalMQProducer;
|
||||
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
|
||||
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
|
||||
|
||||
public class CanalRocketMQProducer implements CanalMQProducer {
|
||||
|
||||
@@ -34,8 +35,8 @@ public class CanalRocketMQProducer implements CanalMQProducer {
|
||||
public void init(MQProperties rocketMQProperties) {
|
||||
this.mqProperties = rocketMQProperties;
|
||||
RPCHook rpcHook = null;
|
||||
if(rocketMQProperties.getAliyunAccessKey().length() > 0
|
||||
&& rocketMQProperties.getAliyunSecretKey().length() > 0){
|
||||
if (rocketMQProperties.getAliyunAccessKey().length() > 0
|
||||
&& rocketMQProperties.getAliyunSecretKey().length() > 0) {
|
||||
SessionCredentials sessionCredentials = new SessionCredentials();
|
||||
sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
|
||||
sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
|
||||
@@ -59,23 +60,65 @@ public class CanalRocketMQProducer implements CanalMQProducer {
|
||||
Callback callback) {
|
||||
if (!mqProperties.getFlatMessage()) {
|
||||
try {
|
||||
Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data,
|
||||
mqProperties.isFilterTransactionEntry()));
|
||||
logger.debug("send message:{} to destination:{}, partition: {}",
|
||||
message,
|
||||
destination.getCanalDestination(),
|
||||
destination.getPartition());
|
||||
this.defaultMQProducer.send(message, new MessageQueueSelector() {
|
||||
|
||||
@Override
|
||||
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
|
||||
int partition = 0;
|
||||
if (destination.getPartition() != null) {
|
||||
partition = destination.getPartition();
|
||||
}
|
||||
return mqs.get(partition);
|
||||
if (destination.getPartition() != null) {
|
||||
Message message = new Message(destination.getTopic(),
|
||||
CanalMessageSerializer.serializer(data, mqProperties.isFilterTransactionEntry()));
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message:{} to destination:{}, partition: {}",
|
||||
message,
|
||||
destination.getCanalDestination(),
|
||||
destination.getPartition());
|
||||
}
|
||||
}, null);
|
||||
this.defaultMQProducer.send(message, new MessageQueueSelector() {
|
||||
|
||||
@Override
|
||||
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
|
||||
int partition = 0;
|
||||
if (destination.getPartition() != null) {
|
||||
partition = destination.getPartition();
|
||||
}
|
||||
return mqs.get(partition);
|
||||
}
|
||||
}, null);
|
||||
} else {
|
||||
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
|
||||
com.alibaba.otter.canal.protocol.Message[] messages = data
|
||||
.messagePartition(destination.getPartitionsNum(), destination.getPartitionHash());
|
||||
int length = messages.length;
|
||||
for (int i = 0; i < length; i++) {
|
||||
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
|
||||
if (dataPartition != null) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("flatMessagePart: {}, partition: {}",
|
||||
JSON.toJSONString(dataPartition, SerializerFeature.WriteMapNullValue),
|
||||
i);
|
||||
}
|
||||
final int index = i;
|
||||
try {
|
||||
Message message = new Message(destination.getTopic(),
|
||||
CanalMessageSerializer.serializer(dataPartition,
|
||||
mqProperties.isFilterTransactionEntry()));
|
||||
this.defaultMQProducer.send(message, new MessageQueueSelector() {
|
||||
|
||||
@Override
|
||||
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
|
||||
if (index > mqs.size()) {
|
||||
throw new CanalServerException("partition number is error,config num:"
|
||||
+ destination.getPartitionsNum()
|
||||
+ ", mq num: " + mqs.size());
|
||||
}
|
||||
return mqs.get(index);
|
||||
}
|
||||
}, null);
|
||||
} catch (Exception e) {
|
||||
logger.error("send flat message to hashed partition error", e);
|
||||
callback.rollback();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
|
||||
logger.error("Send message error!", e);
|
||||
callback.rollback();
|
||||
@@ -87,10 +130,12 @@ public class CanalRocketMQProducer implements CanalMQProducer {
|
||||
for (FlatMessage flatMessage : flatMessages) {
|
||||
if (destination.getPartition() != null) {
|
||||
try {
|
||||
logger.info("send flat message: {} to topic: {} fixed partition: {}",
|
||||
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue),
|
||||
destination.getTopic(),
|
||||
destination.getPartition());
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message: {} to topic: {} fixed partition: {}",
|
||||
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue),
|
||||
destination.getTopic(),
|
||||
destination.getPartition());
|
||||
}
|
||||
Message message = new Message(destination.getTopic(),
|
||||
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue).getBytes());
|
||||
this.defaultMQProducer.send(message, new MessageQueueSelector() {
|
||||
@@ -114,13 +159,16 @@ public class CanalRocketMQProducer implements CanalMQProducer {
|
||||
for (int i = 0; i < length; i++) {
|
||||
FlatMessage flatMessagePart = partitionFlatMessage[i];
|
||||
if (flatMessagePart != null) {
|
||||
logger.debug("flatMessagePart: {}, partition: {}",
|
||||
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue),
|
||||
i);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("flatMessagePart: {}, partition: {}",
|
||||
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue),
|
||||
i);
|
||||
}
|
||||
final int index = i;
|
||||
try {
|
||||
Message message = new Message(destination.getTopic(),
|
||||
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue).getBytes());
|
||||
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue)
|
||||
.getBytes());
|
||||
this.defaultMQProducer.send(message, new MessageQueueSelector() {
|
||||
|
||||
@Override
|
||||
|
||||
Reference in New Issue
Block a user