chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加扁平Message用于卡夫卡数据投递

Browse Source
This commit is contained in:
mcy
2018-09-14 11:22:47 +08:00
parent a6798c61df
commit e8e0032404
19 changed files with 686 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.kafka;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.Future;
@@ -11,6 +12,8 @@ import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -25,7 +28,12 @@ public class CanalKafkaProducer {
private Producer<String, Message> producer;
private Producer<String, String> producer2; // 用于扁平message的数据投递
private KafkaProperties kafkaProperties;
public void init(KafkaProperties kafkaProperties) {
this.kafkaProperties = kafkaProperties;
Properties properties = new Properties();
properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", "all");
@@ -34,15 +42,26 @@ public class CanalKafkaProducer {
properties.put("linger.ms", kafkaProperties.getLingerMs());
properties.put("buffer.memory", kafkaProperties.getBufferMemory());
properties.put("key.serializer", StringSerializer.class.getName());
properties.put("value.serializer", MessageSerializer.class.getName());
producer = new KafkaProducer<String, Message>(properties);
if (!kafkaProperties.getFlatMessage()) {
properties.put("value.serializer", MessageSerializer.class.getName());
producer = new KafkaProducer<String, Message>(properties);
} else {
properties.put("value.serializer", StringSerializer.class.getName());
producer2 = new KafkaProducer<String, String>(properties);
}
// producer.initTransactions();
}
public void stop() {
try {
logger.info("## stop the kafka producer");
producer.close();
if (producer != null) {
producer.close();
}
if (producer2 != null) {
producer2.close();
}
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka producer:", e);
} finally {
@@ -53,14 +72,27 @@ public class CanalKafkaProducer {
public void send(KafkaProperties.Topic topic, Message message, Callback callback) {
try {
// producer.beginTransaction();
ProducerRecord<String, Message> record;
if (topic.getPartition() != null) {
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
if (!kafkaProperties.getFlatMessage()) {
ProducerRecord<String, Message> record;
if (topic.getPartition() != null) {
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
} else {
record = new ProducerRecord<String, Message>(topic.getTopic(), message);
}
Future<RecordMetadata> future = producer.send(record);
future.get();
} else {
record = new ProducerRecord<String, Message>(topic.getTopic(), message);
// 发送扁平数据json
List<FlatMessage> flatMessages = FlatMessage.messageConverter(message);
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(topic.getTopic(),
JSON.toJSONString(flatMessage));
producer2.send(record);
}
}
}
Future<RecordMetadata> future = producer.send(record);
future.get();
// producer.commitTransaction();
callback.commit();
if (logger.isDebugEnabled()) {

9
server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java
View File

@@ -21,6 +21,7 @@ public class KafkaProperties {
private boolean filterTransactionEntry = true;
private int canalBatchSize = 50;
private Long canalGetTimeout;
private boolean flatMessage = true;
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
@@ -160,6 +161,14 @@ public class KafkaProperties {
this.canalGetTimeout = canalGetTimeout;
}
public boolean getFlatMessage() {
return flatMessage;
}
public void setFlatMessage(boolean flatMessage) {
this.flatMessage = flatMessage;
}
public List<CanalDestination> getCanalDestinations() {
return canalDestinations;
}

7
server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
View File

@@ -311,9 +311,10 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
}
if (CollectionUtils.isEmpty(events.getEvents())) {
logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result is null",
clientIdentity.getClientId(),
batchSize);
// logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result
// is null",
// clientIdentity.getClientId(),
// batchSize);
return new Message(-1, true, new ArrayList()); // 返回空包，避免生成batchId，浪费性能
} else {
// 记录到流式信息