chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

format & import

Browse Source
This commit is contained in:
七锋
2018-09-30 10:52:41 +08:00
parent c3327e567e
commit 9344ed121f
46 changed files with 458 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java
View File

@@ -1,5 +1,10 @@
package com.alibaba.otter.canal.common;
import java.util.List;
import org.apache.kafka.common.errors.SerializationException;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
@@ -7,12 +12,11 @@ import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
import java.util.List;
import org.apache.kafka.common.errors.SerializationException;
import org.springframework.util.CollectionUtils;
public class CanalMessageSerializer {
public static byte[] serializer(Message data){
@SuppressWarnings("deprecation")
public static byte[] serializer(Message data) {
try {
if (data != null) {
if (data.getId() != -1) {
@@ -31,11 +35,9 @@ public class CanalMessageSerializer {
messageSize += 1 * rowEntries.size();
// packet size
int size = 0;
size += CodedOutputStream.computeEnumSize(3,
PacketType.MESSAGES.getNumber());
size += CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
size += CodedOutputStream.computeTagSize(5)
+ CodedOutputStream.computeRawVarint32Size(messageSize)
+ messageSize;
+ CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
// build data
byte[] body = new byte[size];
CodedOutputStream output = CodedOutputStream.newInstance(body);

31
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -12,25 +12,25 @@ import java.util.Map;
*/
public class MQProperties {
private String servers = "localhost:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout;
private boolean flatMessage = true;
private String servers = "localhost:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout;
private boolean flatMessage = true;
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
public static class CanalDestination {
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private Map<String, String> partitionHash;
public String getCanalDestination() {
@@ -74,7 +74,6 @@ public class MQProperties {
}
}
public String getServers() {
return servers;
}

31
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -1,12 +1,8 @@
package com.alibaba.otter.canal.kafka;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import java.util.List;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
@@ -14,6 +10,13 @@ import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer;
;
/**
* kafka producer 主操作类
*
@@ -22,13 +25,13 @@ import org.slf4j.LoggerFactory;
*/
public class CanalKafkaProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private Producer<String, Message> producer;
private Producer<String, String> producer2; // 用于扁平message的数据投递
private Producer<String, String> producer2; // 用于扁平message的数据投递
private MQProperties kafkaProperties;
private MQProperties kafkaProperties;
@Override
public void init(MQProperties kafkaProperties) {
@@ -91,8 +94,10 @@ public class CanalKafkaProducer implements CanalMQProducer {
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartition() != null) {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination
.getTopic(), canalDestination.getPartition(), null, JSON.toJSONString(flatMessage));
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(),
canalDestination.getPartition(),
null,
JSON.toJSONString(flatMessage));
producer2.send(record);
} else {
if (canalDestination.getPartitionHash() != null
@@ -104,8 +109,7 @@ public class CanalKafkaProducer implements CanalMQProducer {
for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
canalDestination.getTopic(),
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(),
i,
null,
JSON.toJSONString(flatMessagePart));
@@ -113,8 +117,7 @@ public class CanalKafkaProducer implements CanalMQProducer {
}
}
} else {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
canalDestination.getTopic(),
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination.getTopic(),
0,
null,
JSON.toJSONString(flatMessage));

6
server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java
View File

@@ -1,9 +1,11 @@
package com.alibaba.otter.canal.kafka;
import java.util.Map;
import org.apache.kafka.common.serialization.Serializer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.protocol.Message;
import java.util.Map;
import org.apache.kafka.common.serialization.Serializer;
/**
* Kafka Message类的序列化

6
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
View File

@@ -16,9 +16,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CanalRocketMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
private DefaultMQProducer defaultMQProducer;
@Override
public void init(MQProperties rocketMQProperties) {
@@ -36,10 +37,11 @@ public class CanalRocketMQProducer implements CanalMQProducer {
@Override
public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
Callback callback) {
Callback callback) {
try {
Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data));
this.defaultMQProducer.send(message, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
int partition = 0;

35
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -1,35 +1,38 @@
package com.alibaba.otter.canal.server;
import java.io.FileInputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import java.io.FileInputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
public class CanalMQStarter {
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private volatile boolean running = false;
private volatile boolean running = false;
private ExecutorService executorService;
private ExecutorService executorService;
private CanalMQProducer canalMQProducer;
private CanalMQProducer canalMQProducer;
private MQProperties properties;
private MQProperties properties;
public CanalMQStarter(CanalMQProducer canalMQProducer) {
public CanalMQStarter(CanalMQProducer canalMQProducer){
this.canalMQProducer = canalMQProducer;
}
@@ -47,7 +50,7 @@ public class CanalMQStarter {
}
// 初始化 kafka producer
// canalMQProducer = new CanalKafkaProducer();
// canalMQProducer = new CanalKafkaProducer();
canalMQProducer.init(properties);
// set filterTransactionEntry
if (properties.isFilterTransactionEntry()) {

2
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
View File

@@ -50,7 +50,7 @@ public class SessionHandler extends SimpleChannelHandler {
this.embeddedServer = embeddedServer;
}
@SuppressWarnings("deprecation")
@SuppressWarnings({ "deprecation", "deprecation" })
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
logger.info("message receives in session handler...");
long start = System.nanoTime();

9
server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java
View File

@@ -1,11 +1,12 @@
package com.alibaba.otter.canal.spi;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.protocol.Message;
import java.io.IOException;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.Message;
public interface CanalMQProducer {
/**
* Init producer.
*
@@ -33,4 +34,4 @@ public interface CanalMQProducer {
void rollback();
}
}
}