chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed format

Browse Source
This commit is contained in:
七锋
2018-06-14 19:25:34 +08:00
parent dca1b20043
commit a4255a6377
38 changed files with 11024 additions and 11392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalLauncher.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.kafka;
import com.alibaba.otter.canal.kafka.producer.CanalKafkaStarter;
/**
@@ -11,9 +10,8 @@ import com.alibaba.otter.canal.kafka.producer.CanalKafkaStarter;
*/
public class CanalLauncher {
public static void main(String[] args) {
public static void main(String[] args) {
CanalServerStarter.init();
CanalKafkaStarter.init();
}
}
}

16
kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java
View File

@@ -1,12 +1,13 @@
package com.alibaba.otter.canal.kafka;
import com.alibaba.otter.canal.deployer.CanalController;
import java.io.FileInputStream;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.util.Properties;
import com.alibaba.otter.canal.deployer.CanalController;
/**
* canal server 启动类
@@ -15,10 +16,10 @@ import java.util.Properties;
* @version 1.0.0
*/
public class CanalServerStarter {
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class);
private static boolean running = false;
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class);
private static boolean running = false;
public static void init() {
try {
@@ -61,12 +62,13 @@ public class CanalServerStarter {
}
}
public static boolean isRunning(){
public static boolean isRunning() {
return running;
}
private static void setGlobalUncaughtExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
logger.error("UnCaughtException", e);

16
kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
View File

@@ -1,8 +1,7 @@
package com.alibaba.otter.canal.kafka.producer;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
@@ -10,7 +9,9 @@ import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
/**
* kafka producer 主操作类
@@ -19,7 +20,8 @@ import java.util.Properties;
* @version 1.0.0
*/
public class CanalKafkaProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private Producer<String, Message> producer;
@@ -51,7 +53,8 @@ public class CanalKafkaProducer {
boolean valid = false;
if (message != null && !message.getEntries().isEmpty()) {
for (CanalEntry.Entry entry : message.getEntries()) {
if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN
&& entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) {
valid = true;
break;
}
@@ -61,7 +64,6 @@ public class CanalKafkaProducer {
return;
}
ProducerRecord<String, Message> record;
if (topic.getPartition() != null) {
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);

52
kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
View File

@@ -1,20 +1,21 @@
package com.alibaba.otter.canal.kafka.producer;
import java.io.FileInputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.kafka.CanalServerStarter;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.CanalDestination;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import java.io.FileInputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* kafka 启动类
@@ -23,36 +24,36 @@ import java.util.concurrent.Executors;
* @version 1.0.0
*/
public class CanalKafkaStarter {
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaStarter.class);
private volatile static boolean running = false;
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaStarter.class);
private static ExecutorService executorService;
private volatile static boolean running = false;
private static ExecutorService executorService;
private static CanalKafkaProducer canalKafkaProducer;
private static KafkaProperties kafkaProperties;
private static KafkaProperties kafkaProperties;
public static void init() {
try {
logger.info("## load kafka configurations");
String conf = System.getProperty("kafka.conf", "classpath:kafka.yml");
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
kafkaProperties = new Yaml().loadAs(CanalKafkaStarter.class.getClassLoader().getResourceAsStream(conf), KafkaProperties.class);
kafkaProperties = new Yaml().loadAs(CanalKafkaStarter.class.getClassLoader().getResourceAsStream(conf),
KafkaProperties.class);
} else {
kafkaProperties = new Yaml().loadAs(new FileInputStream(conf), KafkaProperties.class);
}
//初始化 kafka producer
// 初始化 kafka producer
canalKafkaProducer = new CanalKafkaProducer();
canalKafkaProducer.init(kafkaProperties);
//对应每个instance启动一个worker线程
// 对应每个instance启动一个worker线程
List<CanalDestination> destinations = kafkaProperties.getCanalDestinations();
executorService = Executors.newFixedThreadPool(destinations.size());
@@ -60,6 +61,7 @@ public class CanalKafkaStarter {
logger.info("## start the kafka workers.");
for (final CanalDestination destination : destinations) {
executorService.execute(new Runnable() {
@Override
public void run() {
worker(destination);
@@ -69,6 +71,7 @@ public class CanalKafkaStarter {
running = true;
logger.info("## the kafka workers is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka workers");
@@ -90,10 +93,11 @@ public class CanalKafkaStarter {
}
}
private static void worker(CanalDestination destination) {
while (!running) ;
while (!CanalServerStarter.isRunning()) ; //等待server启动完成
while (!running)
;
while (!CanalServerStarter.isRunning())
; // 等待server启动完成
logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
@@ -103,7 +107,7 @@ public class CanalKafkaStarter {
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
//ignore
// ignore
}
continue;
}
@@ -123,7 +127,7 @@ public class CanalKafkaStarter {
destination.getTopics().add(topic);
}
for (Topic topic : destination.getTopics()) {
canalKafkaProducer.send(topic, message); //发送message到所有topic
canalKafkaProducer.send(topic, message); // 发送message到所有topic
}
}

23
kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java
View File

@@ -12,20 +12,22 @@ import java.util.Set;
* @version 1.0.0
*/
public class KafkaProperties {
private String servers = "localhost:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private long bufferMemory = 33554432L;
private int canalBatchSize = 5;
private String servers = "localhost:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private long bufferMemory = 33554432L;
private int canalBatchSize = 5;
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
public static class CanalDestination {
private String canalDestination;
private String topic;
private Integer partition;
private String canalDestination;
private String topic;
private Integer partition;
private Set<Topic> topics = new HashSet<Topic>();
public String getCanalDestination() {
@@ -62,7 +64,8 @@ public class KafkaProperties {
}
public static class Topic {
private String topic;
private String topic;
private Integer partition;
public String getTopic() {

13
kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java
View File

@@ -1,13 +1,14 @@
package com.alibaba.otter.canal.kafka.producer;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.Message;
import java.util.Map;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.serialization.Serializer;
import org.springframework.util.CollectionUtils;
import java.util.Map;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka Message类的序列化
@@ -16,6 +17,7 @@ import java.util.Map;
* @version 1.0.0
*/
public class MessageSerializer implements Serializer<Message> {
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
}
@@ -23,8 +25,7 @@ public class MessageSerializer implements Serializer<Message> {
@Override
public byte[] serialize(String topic, Message data) {
try {
if (data == null)
return null;
if (data == null) return null;
else {
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
if (data.getId() != -1 && !CollectionUtils.isEmpty(data.getEntries())) {