chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加kafka producer sender的callback，canal和kafka同步提交回滚

Browse Source 
去除一个canal实例对应多个kafka topic的机制，一个实例只对应一个topic
kafka模式下server端的批次大小不能超过1M，最大为900K
This commit is contained in:
mcy
2018-09-03 17:40:30 +08:00
parent ee5775ce05
commit ebf9139762
6 changed files with 66 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,11 +1,11 @@
package com.alibaba.otter.canal.kafka;
import java.io.IOException;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,16 +48,32 @@ public class CanalKafkaProducer {
}
}
public void send(KafkaProperties.Topic topic, Message message) {
ProducerRecord<String, Message> record;
if (topic.getPartition() != null) {
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
} else {
record = new ProducerRecord<String, Message>(topic.getTopic(), message);
}
producer.send(record);
if (logger.isDebugEnabled()) {
logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
public void send(KafkaProperties.Topic topic, Message message, Callback callback) {
producer.initTransactions();
try {
producer.beginTransaction();
ProducerRecord<String, Message> record;
if (topic.getPartition() != null) {
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
} else {
record = new ProducerRecord<String, Message>(topic.getTopic(), message);
}
producer.send(record);
producer.commitTransaction();
callback.commit();
if (logger.isDebugEnabled()) {
logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
}
} catch (KafkaException e) {
producer.abortTransaction();
callback.rollback();
}
}
public interface Callback {
void commit();
void rollback();
}
}

40
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java
View File

@@ -33,9 +33,9 @@ public class CanalKafkaStarter implements CanalServerStarter {
private ExecutorService executorService;
private CanalKafkaProducer canalKafkaProducer;
private CanalKafkaProducer canalKafkaProducer;
private KafkaProperties kafkaProperties;
private KafkaProperties kafkaProperties;
public void init() {
try {
@@ -55,7 +55,7 @@ public class CanalKafkaStarter implements CanalServerStarter {
canalKafkaProducer.init(kafkaProperties);
// set filterTransactionEntry
// if (kafkaProperties.isFilterTransactionEntry()) {
// System.setProperty("canal.instance.filter.transaction.entry", "true");
// System.setProperty("canal.instance.filter.transaction.entry", "true");
// }
// 对应每个instance启动一个worker线程
List<CanalDestination> destinations = kafkaProperties.getCanalDestinations();
@@ -101,8 +101,8 @@ public class CanalKafkaStarter implements CanalServerStarter {
while (!running)
;
logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
final CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
final ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
while (running) {
try {
if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
@@ -118,26 +118,28 @@ public class CanalKafkaStarter implements CanalServerStarter {
while (running) {
Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据
long batchId = message.getId();
final long batchId = message.getId();
try {
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) {
if (!StringUtils.isEmpty(destination.getTopic())) {
Topic topic = new Topic();
topic.setTopic(destination.getTopic());
topic.setPartition(destination.getPartition());
destination.getTopics().add(topic);
}
for (Topic topic : destination.getTopics()) {
canalKafkaProducer.send(topic, message); // 发送message到所有topic
}
Topic topic = new Topic();
topic.setTopic(destination.getTopic());
topic.setPartition(destination.getPartition());
canalKafkaProducer.send(topic, message, new CanalKafkaProducer.Callback() {
@Override
public void commit() {
server.ack(clientIdentity, batchId); // 提交确认
}
@Override
public void rollback() {
server.rollback(clientIdentity, batchId);
}
}); // 发送message到topic
}
if (batchId != -1) {
server.ack(clientIdentity, batchId); // 提交确认
}
} catch (Exception e) {
server.rollback(clientIdentity);
logger.error(e.getMessage(), e);
}
}