chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

调整kafka同步发送和消费，确保消息不丢失

Browse Source
This commit is contained in:
mcy
2018-10-03 16:32:08 +08:00
parent 2e92ec3e4d
commit e4b6385dcc
3 changed files with 99 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java
View File

@@ -96,7 +96,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
while (!running) while (!running)
; ;
ExecutorService executor = Executors.newSingleThreadExecutor(); ExecutorService executor = Executors.newSingleThreadExecutor();
final AtomicBoolean executing = new AtomicBoolean(true); // final AtomicBoolean executing = new AtomicBoolean(true);
while (running) { while (running) {
try { try {
logger.info("=============> Start to connect topic: {} <=============", this.topic); logger.info("=============> Start to connect topic: {} <=============", this.topic);
@@ -116,36 +116,41 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
} }
if (messages != null) { if (messages != null) {
for (final Object message : messages) { for (final Object message : messages) {
executing.set(true);
if (message != null) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
if (message instanceof FlatMessage) { if (message instanceof FlatMessage) {
writeOut((FlatMessage) message); writeOut((FlatMessage) message);
} else { } else {
writeOut((Message) message); writeOut((Message) message);
} }
} catch (Exception e) { // executing.set(true);
logger.error(e.getMessage(), e); // if (message != null) {
} finally { // executor.submit(new Runnable() {
executing.compareAndSet(true, false); //
} // @Override
} // public void run() {
}); // try {
// if (message instanceof FlatMessage) {
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端 // writeOut((FlatMessage) message);
long currentTS = System.currentTimeMillis(); // } else {
while (executing.get()) { // writeOut((Message) message);
// 大于10秒未消费完ack一次keep alive // }
if (System.currentTimeMillis() - currentTS > 10000) { // } catch (Exception e) {
connector.ack(); // logger.error(e.getMessage(), e);
currentTS = System.currentTimeMillis(); // } finally {
} // executing.compareAndSet(true, false);
} // }
} // }
// });
//
// // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
// long currentTS = System.currentTimeMillis();
// while (executing.get()) {
// // 大于10秒未消费完ack一次keep alive
// if (System.currentTimeMillis() - currentTS > 10000) {
// connector.ack();
// currentTS = System.currentTimeMillis();
// }
// }
// }
} }
} }
connector.ack(); connector.ack();

2
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -233,7 +233,7 @@ public class KafkaCanalConnector {
kafkaConsumer.commitSync(); kafkaConsumer.commitSync();
} }
if (kafkaConsumer2 != null) { if (kafkaConsumer2 != null) {
kafkaConsumer2.commitAsync(); kafkaConsumer2.commitSync();
} }
} }

57
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,11 +1,14 @@
package com.alibaba.otter.canal.kafka; package com.alibaba.otter.canal.kafka;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.Future;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -14,8 +17,7 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.common.MQProperties; import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer; import com.alibaba.otter.canal.spi.CanalMQProducer;;
;
/** /**
* kafka producer 主操作类 * kafka producer 主操作类
@@ -74,9 +76,10 @@ public class CanalKafkaProducer implements CanalMQProducer {
@Override @Override
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) { public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
try {
// producer.beginTransaction(); // producer.beginTransaction();
if (!kafkaProperties.getFlatMessage()) { if (!kafkaProperties.getFlatMessage()) {
try {
ProducerRecord<String, Message> record; ProducerRecord<String, Message> record;
if (canalDestination.getPartition() != null) { if (canalDestination.getPartition() != null) {
record = new ProducerRecord<String, Message>(canalDestination.getTopic(), record = new ProducerRecord<String, Message>(canalDestination.getTopic(),
@@ -87,18 +90,27 @@ public class CanalKafkaProducer implements CanalMQProducer {
record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message); record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message);
} }
producer.send(record); producer.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
}
} else { } else {
// 发送扁平数据json // 发送扁平数据json
List<FlatMessage> flatMessages = FlatMessage.messageConverter(message); List<FlatMessage> flatMessages = FlatMessage.messageConverter(message);
if (flatMessages != null) { if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) { for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartition() != null) { if (canalDestination.getPartition() != null) {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(), try {
canalDestination.getPartition(), ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination
null, .getTopic(), canalDestination.getPartition(), null, JSON.toJSONString(flatMessage));
JSON.toJSONString(flatMessage));
producer2.send(record); producer2.send(record);
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
}
} else { } else {
if (canalDestination.getPartitionHash() != null if (canalDestination.getPartitionHash() != null
&& !canalDestination.getPartitionHash().isEmpty()) { && !canalDestination.getPartitionHash().isEmpty()) {
@@ -109,22 +121,35 @@ public class CanalKafkaProducer implements CanalMQProducer {
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i]; FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) { if (flatMessagePart != null) {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(), try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
canalDestination.getTopic(),
i, i,
null, null,
JSON.toJSONString(flatMessagePart)); JSON.toJSONString(flatMessagePart));
producer2.send(record); producer2.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
}
} }
} }
} else { } else {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(), try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
canalDestination.getTopic(),
0, 0,
null, null,
JSON.toJSONString(flatMessage)); JSON.toJSONString(flatMessage));
producer2.send(record); producer2.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
}
} }
} }
} }
} }
} }
@@ -134,11 +159,7 @@ public class CanalKafkaProducer implements CanalMQProducer {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("send message to kafka topic: {}", canalDestination.getTopic()); logger.debug("send message to kafka topic: {}", canalDestination.getTopic());
} }
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
}
} }
} }