fixed issue #1075, support message transactor filter

This commit is contained in:
七锋
2018-10-30 23:48:05 +08:00
parent be5f1546ca
commit dd8243a56d
3 changed files with 18 additions and 3 deletions

View File

@@ -16,7 +16,7 @@ import com.google.protobuf.WireFormat;
public class CanalMessageSerializer { public class CanalMessageSerializer {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
public static byte[] serializer(Message data) { public static byte[] serializer(Message data, boolean filterTransactionEntry) {
try { try {
if (data != null) { if (data != null) {
if (data.getId() != -1) { if (data.getId() != -1) {
@@ -53,8 +53,14 @@ public class CanalMessageSerializer {
output.checkNoSpaceLeft(); output.checkNoSpaceLeft();
return body; return body;
} else if (!CollectionUtils.isEmpty(data.getEntries())) { } else if (!CollectionUtils.isEmpty(data.getEntries())) {
// mq模式只会走到非rowEntry模式
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) { for (CanalEntry.Entry entry : data.getEntries()) {
if (filterTransactionEntry
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
continue;
}
messageBuilder.addMessages(entry.toByteString()); messageBuilder.addMessages(entry.toByteString());
} }

View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.kafka;
import java.util.Map; import java.util.Map;
import org.apache.commons.lang.BooleanUtils;
import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.Serializer;
import com.alibaba.otter.canal.common.CanalMessageSerializer; import com.alibaba.otter.canal.common.CanalMessageSerializer;
@@ -15,13 +16,20 @@ import com.alibaba.otter.canal.protocol.Message;
*/ */
public class MessageSerializer implements Serializer<Message> { public class MessageSerializer implements Serializer<Message> {
private boolean filterTransactionEntry = false;
public MessageSerializer(){
this.filterTransactionEntry = BooleanUtils.toBoolean(System.getProperty("canal.instance.filter.transaction.entry",
"false"));
}
@Override @Override
public void configure(Map<String, ?> configs, boolean isKey) { public void configure(Map<String, ?> configs, boolean isKey) {
} }
@Override @Override
public byte[] serialize(String topic, Message data) { public byte[] serialize(String topic, Message data) {
return CanalMessageSerializer.serializer(data); return CanalMessageSerializer.serializer(data, filterTransactionEntry);
} }
@Override @Override

View File

@@ -47,7 +47,8 @@ public class CanalRocketMQProducer implements CanalMQProducer {
Callback callback) { Callback callback) {
if (!mqProperties.getFlatMessage()) { if (!mqProperties.getFlatMessage()) {
try { try {
Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data)); Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data,
mqProperties.isFilterTransactionEntry()));
logger.debug("send message:{} to destination:{}, partition: {}", logger.debug("send message:{} to destination:{}, partition: {}",
message, message,
destination.getCanalDestination(), destination.getCanalDestination(),