chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add RocketMQ canal connector

Browse Source
This commit is contained in:
duhengforever
2018-09-20 15:48:52 +08:00
parent 00b5ade787
commit d5aa324ce4
24 changed files with 881 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
server/pom.xml
View File

@@ -42,6 +42,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
</dependency>
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
<dependency>
<groupId>org.jboss.netty</groupId>

70
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java Normal file
View File

@@ -0,0 +1,70 @@
package com.alibaba.otter.canal.common;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
import java.util.List;
import org.apache.kafka.common.errors.SerializationException;
import org.springframework.util.CollectionUtils;
public class CanalMessageSerializer {
public static byte[] serializer(Message data){
try {
if (data != null) {
if (data.getId() != -1) {
if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
// for performance
List<ByteString> rowEntries = data.getRawEntries();
// message size
int messageSize = 0;
messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
int dataSize = 0;
for (int i = 0; i < rowEntries.size(); i++) {
dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
}
messageSize += dataSize;
messageSize += 1 * rowEntries.size();
// packet size
int size = 0;
size += CodedOutputStream.computeEnumSize(3,
CanalPacket.PacketType.MESSAGES.getNumber());
size += CodedOutputStream.computeTagSize(5)
+ CodedOutputStream.computeRawVarint32Size(messageSize)
+ messageSize;
// build data
byte[] body = new byte[size];
CodedOutputStream output = CodedOutputStream.newInstance(body);
output.writeEnum(3, CanalPacket.PacketType.MESSAGES.getNumber());
output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
output.writeRawVarint32(messageSize);
// message
output.writeInt64(1, data.getId());
for (int i = 0; i < rowEntries.size(); i++) {
output.writeBytes(2, rowEntries.get(i));
}
output.checkNoSpaceLeft();
return body;
} else if (!CollectionUtils.isEmpty(data.getEntries())) {
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) {
messageBuilder.addMessages(entry.toByteString());
}
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(CanalPacket.PacketType.MESSAGES);
packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray();
}
}
}
} catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] ");
}
return null;
}
}

12
server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java → server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -1,4 +1,4 @@
package com.alibaba.otter.canal.kafka;
package com.alibaba.otter.canal.common;
import java.util.ArrayList;
import java.util.HashSet;
@@ -11,7 +11,7 @@ import java.util.Set;
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class KafkaProperties {
public class MQProperties {
private String servers = "localhost:6667";
private int retries = 0;
@@ -20,6 +20,7 @@ public class KafkaProperties {
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private int canalBatchSize = 5;
private String producerGroup = "Canal-Producer";
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
@@ -167,4 +168,11 @@ public class KafkaProperties {
this.filterTransactionEntry = filterTransactionEntry;
}
public String getProducerGroup() {
return producerGroup;
}
public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup;
}
}

39
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.kafka;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import java.io.IOException;
import java.util.Properties;
@@ -18,13 +20,14 @@ import com.alibaba.otter.canal.protocol.Message;
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class CanalKafkaProducer {
public class CanalKafkaProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private Producer<String, Message> producer;
public void init(KafkaProperties kafkaProperties) {
@Override
public void init(MQProperties kafkaProperties) {
Properties properties = new Properties();
properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", "all");
@@ -37,6 +40,7 @@ public class CanalKafkaProducer {
producer = new KafkaProducer<String, Message>(properties);
}
@Override
public void stop() {
try {
logger.info("## stop the kafka producer");
@@ -48,33 +52,8 @@ public class CanalKafkaProducer {
}
}
public void send(KafkaProperties.Topic topic, Message message) throws IOException {
// set canal.instance.filter.transaction.entry = true
// boolean valid = false;
// if (message != null) {
// if (message.isRaw() && !message.getRawEntries().isEmpty()) {
// for (ByteString byteString : message.getRawEntries()) {
// CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
// if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
// && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
// valid = true;
// break;
// }
// }
// } else if (!message.getEntries().isEmpty()){
// for (CanalEntry.Entry entry : message.getEntries()) {
// if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
// && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
// valid = true;
// break;
// }
// }
// }
// }
// if (!valid) {
// return;
// }
@Override
public void send(MQProperties.Topic topic, Message message) throws IOException {
ProducerRecord<String, Message> record;
if (topic.getPartition() != null) {
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);

56
server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.kafka;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import java.util.List;
import java.util.Map;
@@ -29,60 +30,7 @@ public class MessageSerializer implements Serializer<Message> {
@Override
public byte[] serialize(String topic, Message data) {
try {
if (data != null) {
if (data.getId() != -1) {
if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
// for performance
List<ByteString> rowEntries = data.getRawEntries();
// message size
int messageSize = 0;
messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
int dataSize = 0;
for (int i = 0; i < rowEntries.size(); i++) {
dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
}
messageSize += dataSize;
messageSize += 1 * rowEntries.size();
// packet size
int size = 0;
size += CodedOutputStream.computeEnumSize(3,
PacketType.MESSAGES.getNumber());
size += CodedOutputStream.computeTagSize(5)
+ CodedOutputStream.computeRawVarint32Size(messageSize)
+ messageSize;
// build data
byte[] body = new byte[size];
CodedOutputStream output = CodedOutputStream.newInstance(body);
output.writeEnum(3, PacketType.MESSAGES.getNumber());
output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
output.writeRawVarint32(messageSize);
// message
output.writeInt64(1, data.getId());
for (int i = 0; i < rowEntries.size(); i++) {
output.writeBytes(2, rowEntries.get(i));
}
output.checkNoSpaceLeft();
return body;
} else if (!CollectionUtils.isEmpty(data.getEntries())) {
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) {
messageBuilder.addMessages(entry.toByteString());
}
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray();
}
}
}
} catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] ");
}
return null;
return CanalMessageSerializer.serializer(data);
}
@Override

58
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java Normal file
View File

@@ -0,0 +1,58 @@
package com.alibaba.otter.canal.rocketmq;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import java.util.List;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CanalRocketMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
@Override
public void init(MQProperties rocketMQProperties) {
defaultMQProducer = new DefaultMQProducer();
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
defaultMQProducer.setProducerGroup(rocketMQProperties.getProducerGroup());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
logger.info("##Start RocketMQ producer##");
try {
defaultMQProducer.start();
} catch (MQClientException ex) {
throw new CanalServerException("Start RocketMQ producer error", ex);
}
}
@Override
public void send(MQProperties.Topic topic, com.alibaba.otter.canal.protocol.Message data) {
try {
Message message = new Message(topic.getTopic(), CanalMessageSerializer.serializer(data));
this.defaultMQProducer.send(message, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
// int index = (arg.hashCode() % mqs.size());
return mqs.get(1);
}
}, null);
} catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
logger.error("Send message error!", e);
}
}
@Override
public void stop() {
logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown();
}
}

80
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java → server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -1,69 +1,63 @@
package com.alibaba.otter.canal.kafka;
package com.alibaba.otter.canal.server;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import java.io.FileInputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.kafka.KafkaProperties.CanalDestination;
import com.alibaba.otter.canal.kafka.KafkaProperties.Topic;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.CanalServerStarter;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
/**
* kafka 启动类
*
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class CanalKafkaStarter implements CanalServerStarter {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaStarter.class);
public class CanalMQStarter {
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private volatile boolean running = false;
private ExecutorService executorService;
private ExecutorService executorService;
private CanalKafkaProducer canalKafkaProducer;
private CanalMQProducer canalMQProducer;
private KafkaProperties kafkaProperties;
private MQProperties properties;
public CanalMQStarter(CanalMQProducer canalMQProducer){
this.canalMQProducer = canalMQProducer;
}
public void init() {
try {
logger.info("## load kafka configurations");
String conf = System.getProperty("kafka.conf", "classpath:kafka.yml");
logger.info("## load MQ configurations");
String conf = System.getProperty("mq.conf", "classpath:mq.yml");
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
kafkaProperties = new Yaml().loadAs(CanalKafkaStarter.class.getClassLoader().getResourceAsStream(conf),
KafkaProperties.class);
properties = new Yaml().loadAs(CanalMQStarter.class.getClassLoader().getResourceAsStream(conf),
MQProperties.class);
} else {
kafkaProperties = new Yaml().loadAs(new FileInputStream(conf), KafkaProperties.class);
properties = new Yaml().loadAs(new FileInputStream(conf), MQProperties.class);
}
// 初始化 kafka producer
canalKafkaProducer = new CanalKafkaProducer();
canalKafkaProducer.init(kafkaProperties);
// canalMQProducer = new CanalKafkaProducer();
canalMQProducer.init(properties);
// set filterTransactionEntry
if (kafkaProperties.isFilterTransactionEntry()) {
if (properties.isFilterTransactionEntry()) {
System.setProperty("canal.instance.filter.transaction.entry", "true");
}
// 对应每个instance启动一个worker线程
List<CanalDestination> destinations = kafkaProperties.getCanalDestinations();
List<MQProperties.CanalDestination> destinations = properties.getCanalDestinations();
executorService = Executors.newFixedThreadPool(destinations.size());
logger.info("## start the kafka workers.");
for (final CanalDestination destination : destinations) {
logger.info("## start the MQ workers.");
for (final MQProperties.CanalDestination destination : destinations) {
executorService.execute(new Runnable() {
@Override
@@ -73,31 +67,31 @@ public class CanalKafkaStarter implements CanalServerStarter {
});
}
running = true;
logger.info("## the kafka workers is running now ......");
logger.info("## the MQ workers is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka workers");
logger.info("## stop the MQ workers");
running = false;
executorService.shutdown();
canalKafkaProducer.stop();
canalMQProducer.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka workers:", e);
logger.warn("##something goes wrong when stopping MQ workers:", e);
} finally {
logger.info("## canal kafka is down.");
logger.info("## canal MQ is down.");
}
}
});
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the canal kafka workers:", e);
logger.error("## Something goes wrong when starting up the canal MQ workers:", e);
System.exit(0);
}
}
private void worker(CanalDestination destination) {
private void worker(MQProperties.CanalDestination destination) {
while (!running)
;
logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
@@ -117,19 +111,19 @@ public class CanalKafkaStarter implements CanalServerStarter {
logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination());
while (running) {
Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据
Message message = server.getWithoutAck(clientIdentity, properties.getCanalBatchSize()); // 获取指定数量的数据
long batchId = message.getId();
try {
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) {
if (!StringUtils.isEmpty(destination.getTopic())) {
Topic topic = new Topic();
MQProperties.Topic topic = new MQProperties.Topic();
topic.setTopic(destination.getTopic());
topic.setPartition(destination.getPartition());
destination.getTopics().add(topic);
}
for (Topic topic : destination.getTopics()) {
canalKafkaProducer.send(topic, message); // 发送message到所有topic
for (MQProperties.Topic topic : destination.getTopics()) {
canalMQProducer.send(topic, message); // 发送message到所有topic
}
}

28
server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java Normal file
View File

@@ -0,0 +1,28 @@
package com.alibaba.otter.canal.spi;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.Message;
import java.io.IOException;
public interface CanalMQProducer {
/**
* Init producer.
*
* @param mqProperties MQ config
*/
void init(MQProperties mqProperties);
/**
* Send canal message to related topic
*
* @param topic MQ topic
* @param message canal message
* @throws IOException
*/
void send(MQProperties.Topic topic, Message message) throws IOException;
/**
* Stop MQ producer service
*/
void stop();
}