chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed format

Browse Source
This commit is contained in:
七锋
2018-06-14 19:25:34 +08:00
parent dca1b20043
commit a4255a6377
38 changed files with 11024 additions and 11392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/KafkaCanalConnector.java
View File

@@ -1,14 +1,15 @@
package com.alibaba.otter.canal.kafka.client;
import com.alibaba.otter.canal.protocol.Message;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.Message;
/**
* canal kafka 数据操作客户端
@@ -20,14 +21,13 @@ public class KafkaCanalConnector {
private KafkaConsumer<String, Message> kafkaConsumer;
private String topic;
private String topic;
private Integer partition;
private Integer partition;
private Properties properties;
private Properties properties;
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId) {
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId){
this.topic = topic;
this.partition = partition;
@@ -36,10 +36,11 @@ public class KafkaCanalConnector {
properties.put("group.id", groupId);
properties.put("enable.auto.commit", false);
properties.put("auto.commit.interval.ms", "1000");
properties.put("auto.offset.reset", "latest"); //earliest //如果没有offset则从最后的offset开始读
properties.put("request.timeout.ms", "40000"); //必须大于session.timeout.ms的设置
properties.put("session.timeout.ms", "30000"); //默认为30秒
properties.put("max.poll.records", "1"); //所以一次只取一条数据
properties.put("auto.offset.reset", "latest"); // earliest
// //如果没有offset则从最后的offset开始读
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
properties.put("session.timeout.ms", "30000"); // 默认为30秒
properties.put("max.poll.records", "1"); // 所以一次只取一条数据
properties.put("key.deserializer", StringDeserializer.class.getName());
properties.put("value.deserializer", MessageDeserializer.class.getName());
}
@@ -110,8 +111,7 @@ public class KafkaCanalConnector {
* @return
*/
public Message getWithoutAck(Long timeout, TimeUnit unit) {
ConsumerRecords<String, Message> records =
kafkaConsumer.poll(unit.toMillis(timeout)); //基于配置，最多只能poll到一条数据
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据
if (!records.isEmpty()) {
return records.iterator().next().value();

3
kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/KafkaCanalConnectors.java
View File

@@ -7,6 +7,7 @@ package com.alibaba.otter.canal.kafka.client;
* @version 1.0.0
*/
public class KafkaCanalConnectors {
/**
* 创建kafka客户端链接
*
@@ -28,7 +29,7 @@ public class KafkaCanalConnectors {
* @param groupId
* @return
*/
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
return new KafkaCanalConnector(servers, topic, null, groupId);
}
}

17
kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java
View File

@@ -1,15 +1,16 @@
package com.alibaba.otter.canal.kafka.client;
import java.util.Map;
import org.apache.kafka.common.serialization.Deserializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.protobuf.ByteString;
import org.apache.kafka.common.serialization.Deserializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
/**
* Kafka Message类的反序列化
@@ -18,6 +19,7 @@ import java.util.Map;
* @version 1.0.0
*/
public class MessageDeserializer implements Deserializer<Message> {
private static Logger logger = LoggerFactory.getLogger(MessageDeserializer.class);
@Override
@@ -27,8 +29,7 @@ public class MessageDeserializer implements Deserializer<Message> {
@Override
public Message deserialize(String topic, byte[] data) {
try {
if (data == null)
return null;
if (data == null) return null;
else {
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
switch (p.getType()) {
@@ -58,4 +59,4 @@ public class MessageDeserializer implements Deserializer<Message> {
public void close() {
// nothing to do
}
}
}

6
kafka-client/src/test/java/com/alibaba/otter/canal/kafka/client/running/AbstractKafkaTest.java
View File

@@ -10,10 +10,10 @@ import org.junit.Assert;
*/
public abstract class AbstractKafkaTest {
public static String topic = "example";
public static String topic = "example";
public static Integer partition = null;
public static String groupId = "g1";
public static String servers = "slave1:6667,slave2:6667,slave3:6667";
public static String groupId = "g1";
public static String servers = "slave1:6667,slave2:6667,slave3:6667";
public void sleep(long time) {
try {

43
kafka-client/src/test/java/com/alibaba/otter/canal/kafka/client/running/CanalKafkaClientExample.java
View File

@@ -1,14 +1,15 @@
package com.alibaba.otter.canal.kafka.client.running;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnector;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnector;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka client example
@@ -17,28 +18,32 @@ import java.util.concurrent.TimeUnit;
* @version 1.0.0
*/
public class CanalKafkaClientExample {
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
private KafkaCanalConnector connector;
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
private static volatile boolean running = false;
private KafkaCanalConnector connector;
private Thread thread = null;
private static volatile boolean running = false;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientExample(String servers, String topic, Integer partition, String groupId) {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientExample(String servers, String topic, Integer partition, String groupId){
connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId);
}
public static void main(String[] args) {
try {
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.servers,
AbstractKafkaTest.topic, AbstractKafkaTest.partition, AbstractKafkaTest.groupId);
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
@@ -56,7 +61,8 @@ public class CanalKafkaClientExample {
}
});
while (running) ;
while (running)
;
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the kafka consumer:", e);
System.exit(0);
@@ -91,12 +97,13 @@ public class CanalKafkaClientExample {
}
private void process() {
while (!running) ;
while (!running)
;
try {
connector.subscribe();
while (running) {
try {
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); //获取message
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
if (message == null) {
continue;
}
@@ -129,4 +136,4 @@ public class CanalKafkaClientExample {
}
connector.close();
}
}
}

19
kafka-client/src/test/java/com/alibaba/otter/canal/kafka/client/running/KafkaClientRunningTest.java
View File

@@ -1,16 +1,17 @@
package com.alibaba.otter.canal.kafka.client.running;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnector;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnector;
import com.alibaba.otter.canal.kafka.client.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka consumer获取Message的测试例子
@@ -19,7 +20,8 @@ import java.util.concurrent.TimeUnit;
* @version 1.0.0
*/
public class KafkaClientRunningTest extends AbstractKafkaTest {
private Logger logger = LoggerFactory.getLogger(KafkaClientRunningTest.class);
private Logger logger = LoggerFactory.getLogger(KafkaClientRunningTest.class);
private boolean running = true;
@@ -30,6 +32,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
final KafkaCanalConnector connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId);
executor.submit(new Runnable() {
@Override
public void run() {
connector.subscribe();
@@ -41,7 +44,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
}
connector.ack();
} catch (WakeupException e) {
//ignore
// ignore
}
}
connector.unsubscribe();