chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

kafka producer 适配 row data for performance #726

Browse Source
This commit is contained in:
winger
2018-07-31 10:34:19 +08:00
parent fe12a8481f
commit 3d5f13fec8
2 changed files with 34 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
View File

@@ -1,7 +1,9 @@
package com.alibaba.otter.canal.kafka.producer; package com.alibaba.otter.canal.kafka.producer;
import java.io.IOException;
import java.util.Properties; import java.util.Properties;
import com.google.protobuf.ByteString;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.ProducerRecord;
@@ -49,14 +51,25 @@ public class CanalKafkaProducer {
} }
} }
public void send(Topic topic, Message message) { public void send(Topic topic, Message message) throws IOException {
boolean valid = false; boolean valid = false;
if (message != null && !message.getEntries().isEmpty()) { if (message != null) {
for (CanalEntry.Entry entry : message.getEntries()) { if (message.isRaw() && !message.getRawEntries().isEmpty()) {
if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN for (ByteString byteString : message.getRawEntries()) {
&& entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
valid = true; if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
break; && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
valid = true;
break;
}
}
} else if (!message.getEntries().isEmpty()){
for (CanalEntry.Entry entry : message.getEntries()) {
if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
&& entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
valid = true;
break;
}
} }
} }
} }

25
kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java
View File

@@ -1,14 +1,13 @@
package com.alibaba.otter.canal.kafka.producer; package com.alibaba.otter.canal.kafka.producer;
import java.util.Map;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.serialization.Serializer;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket; import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.serialization.Serializer;
import org.springframework.util.CollectionUtils;
import java.util.Map;
/** /**
* Kafka Message类的序列化 * Kafka Message类的序列化
@@ -25,12 +24,15 @@ public class MessageSerializer implements Serializer<Message> {
@Override @Override
public byte[] serialize(String topic, Message data) { public byte[] serialize(String topic, Message data) {
try { try {
if (data == null) return null; if (data != null) {
else {
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
if (data.getId() != -1 && !CollectionUtils.isEmpty(data.getEntries())) { if (data.getId() != -1) {
for (CanalEntry.Entry entry : data.getEntries()) { if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
messageBuilder.addMessages(entry.toByteString()); messageBuilder.addAllMessages(data.getRawEntries());
} else if (!CollectionUtils.isEmpty(data.getEntries())) {
for (CanalEntry.Entry entry : data.getEntries()) {
messageBuilder.addMessages(entry.toByteString());
}
} }
} }
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
@@ -41,6 +43,7 @@ public class MessageSerializer implements Serializer<Message> {
} catch (Exception e) { } catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] "); throw new SerializationException("Error when serializing message to byte[] ");
} }
return null;
} }
@Override @Override