chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

调整kafka同步发送和消费，确保消息不丢失

Browse Source
This commit is contained in:
mcy
2018-10-03 16:32:08 +08:00
parent 2e92ec3e4d
commit e4b6385dcc
3 changed files with 99 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java
View File

@@ -96,7 +96,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
while (!running) while (!running)
; ;
ExecutorService executor = Executors.newSingleThreadExecutor(); ExecutorService executor = Executors.newSingleThreadExecutor();
final AtomicBoolean executing = new AtomicBoolean(true); // final AtomicBoolean executing = new AtomicBoolean(true);
while (running) { while (running) {
try { try {
logger.info("=============> Start to connect topic: {} <=============", this.topic); logger.info("=============> Start to connect topic: {} <=============", this.topic);
@@ -116,36 +116,41 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
} }
if (messages != null) { if (messages != null) {
for (final Object message : messages) { for (final Object message : messages) {
executing.set(true); if (message instanceof FlatMessage) {
if (message != null) { writeOut((FlatMessage) message);
executor.submit(new Runnable() { } else {
writeOut((Message) message);
@Override
public void run() {
try {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
executing.compareAndSet(true, false);
}
}
});
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
long currentTS = System.currentTimeMillis();
while (executing.get()) {
// 大于10秒未消费完ack一次keep alive
if (System.currentTimeMillis() - currentTS > 10000) {
connector.ack();
currentTS = System.currentTimeMillis();
}
}
} }
// executing.set(true);
// if (message != null) {
// executor.submit(new Runnable() {
//
// @Override
// public void run() {
// try {
// if (message instanceof FlatMessage) {
// writeOut((FlatMessage) message);
// } else {
// writeOut((Message) message);
// }
// } catch (Exception e) {
// logger.error(e.getMessage(), e);
// } finally {
// executing.compareAndSet(true, false);
// }
// }
// });
//
// // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
// long currentTS = System.currentTimeMillis();
// while (executing.get()) {
// // 大于10秒未消费完ack一次keep alive
// if (System.currentTimeMillis() - currentTS > 10000) {
// connector.ack();
// currentTS = System.currentTimeMillis();
// }
// }
// }
} }
} }
connector.ack(); connector.ack();

2
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -233,7 +233,7 @@ public class KafkaCanalConnector {
kafkaConsumer.commitSync(); kafkaConsumer.commitSync();
} }
if (kafkaConsumer2 != null) { if (kafkaConsumer2 != null) {
kafkaConsumer2.commitAsync(); kafkaConsumer2.commitSync();
} }
} }

105
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,11 +1,14 @@
package com.alibaba.otter.canal.kafka; package com.alibaba.otter.canal.kafka;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.Future;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -14,8 +17,7 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.common.MQProperties; import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer; import com.alibaba.otter.canal.spi.CanalMQProducer;;
;
/** /**
* kafka producer 主操作类 * kafka producer 主操作类
@@ -74,9 +76,10 @@ public class CanalKafkaProducer implements CanalMQProducer {
@Override @Override
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) { public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
try {
// producer.beginTransaction(); // producer.beginTransaction();
if (!kafkaProperties.getFlatMessage()) { if (!kafkaProperties.getFlatMessage()) {
try {
ProducerRecord<String, Message> record; ProducerRecord<String, Message> record;
if (canalDestination.getPartition() != null) { if (canalDestination.getPartition() != null) {
record = new ProducerRecord<String, Message>(canalDestination.getTopic(), record = new ProducerRecord<String, Message>(canalDestination.getTopic(),
@@ -87,58 +90,76 @@ public class CanalKafkaProducer implements CanalMQProducer {
record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message); record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message);
} }
producer.send(record); producer.send(record).get();
} else { } catch (Exception e) {
// 发送扁平数据json logger.error(e.getMessage(), e);
List<FlatMessage> flatMessages = FlatMessage.messageConverter(message); // producer.abortTransaction();
if (flatMessages != null) { callback.rollback();
for (FlatMessage flatMessage : flatMessages) { }
if (canalDestination.getPartition() != null) { } else {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(), // 发送扁平数据json
canalDestination.getPartition(), List<FlatMessage> flatMessages = FlatMessage.messageConverter(message);
null, if (flatMessages != null) {
JSON.toJSONString(flatMessage)); for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartition() != null) {
try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination
.getTopic(), canalDestination.getPartition(), null, JSON.toJSONString(flatMessage));
producer2.send(record); producer2.send(record);
} else { } catch (Exception e) {
if (canalDestination.getPartitionHash() != null logger.error(e.getMessage(), e);
&& !canalDestination.getPartitionHash().isEmpty()) { // producer.abortTransaction();
FlatMessage[] partitionFlatMessage = FlatMessage.messagePartition(flatMessage, callback.rollback();
canalDestination.getPartitionsNum(), }
canalDestination.getPartitionHash()); } else {
int length = partitionFlatMessage.length; if (canalDestination.getPartitionHash() != null
for (int i = 0; i < length; i++) { && !canalDestination.getPartitionHash().isEmpty()) {
FlatMessage flatMessagePart = partitionFlatMessage[i]; FlatMessage[] partitionFlatMessage = FlatMessage.messagePartition(flatMessage,
if (flatMessagePart != null) { canalDestination.getPartitionsNum(),
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(), canalDestination.getPartitionHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) {
try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
canalDestination.getTopic(),
i, i,
null, null,
JSON.toJSONString(flatMessagePart)); JSON.toJSONString(flatMessagePart));
producer2.send(record); producer2.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
} }
} }
} else { }
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(), } else {
try {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
canalDestination.getTopic(),
0, 0,
null, null,
JSON.toJSONString(flatMessage)); JSON.toJSONString(flatMessage));
producer2.send(record); producer2.send(record).get();
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
} }
} }
} }
} }
} }
// producer.commitTransaction();
callback.commit();
if (logger.isDebugEnabled()) {
logger.debug("send message to kafka topic: {}", canalDestination.getTopic());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
callback.rollback();
} }
// producer.commitTransaction();
callback.commit();
if (logger.isDebugEnabled()) {
logger.debug("send message to kafka topic: {}", canalDestination.getTopic());
}
} }
} }