diff --git a/kafka/pom.xml b/kafka/pom.xml
index 67397a19..5c015e56 100644
--- a/kafka/pom.xml
+++ b/kafka/pom.xml
@@ -18,6 +18,12 @@
${project.version}
+
+ org.yaml
+ snakeyaml
+ 1.17
+
+
org.apache.kafka
@@ -28,6 +34,12 @@
org.apache.kafka
kafka_2.11
0.10.0.1
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
diff --git a/kafka/src/main/assembly/dev.xml b/kafka/src/main/assembly/dev.xml
index df9eca81..2c3e3fff 100644
--- a/kafka/src/main/assembly/dev.xml
+++ b/kafka/src/main/assembly/dev.xml
@@ -34,6 +34,9 @@
**/*
+
+ logback.xml
+
./src/main/resources
diff --git a/kafka/src/main/assembly/release.xml b/kafka/src/main/assembly/release.xml
index d79ecf3c..0b103620 100644
--- a/kafka/src/main/assembly/release.xml
+++ b/kafka/src/main/assembly/release.xml
@@ -34,6 +34,9 @@
**/*
+
+ logback.xml
+
./src/main/resources
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalLauncher.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalLauncher.java
index 0797004e..c38dc968 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalLauncher.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalLauncher.java
@@ -1,13 +1,7 @@
package com.alibaba.otter.canal.kafka;
-import com.alibaba.otter.canal.deployer.CanalController;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileInputStream;
-import java.util.Properties;
+import com.alibaba.otter.canal.kafka.producer.CanalKafkaStarter;
/**
* canal-kafka独立版本启动的入口类
@@ -17,54 +11,9 @@ import java.util.Properties;
*/
public class CanalLauncher {
- private static final String CLASSPATH_URL_PREFIX = "classpath:";
- private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
+ public static void main(String[] args) {
+ CanalServerStarter.init();
- public static void main(String[] args) throws Throwable {
- try {
- logger.info("## set default uncaught exception handler");
- setGlobalUncaughtExceptionHandler();
-
- logger.info("## load canal configurations");
- String conf = System.getProperty("canal.conf", "classpath:canal.properties");
- Properties properties = new Properties();
- if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
- conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
- properties.load(com.alibaba.otter.canal.deployer.CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
- } else {
- properties.load(new FileInputStream(conf));
- }
-
- logger.info("## start the canal server.");
- final CanalController controller = new CanalController(properties);
- controller.start();
- logger.info("## the canal server is running now ......");
- Runtime.getRuntime().addShutdownHook(new Thread() {
-
- public void run() {
- try {
- logger.info("## stop the canal server");
- controller.stop();
- } catch (Throwable e) {
- logger.warn("##something goes wrong when stopping canal Server:", e);
- } finally {
- logger.info("## canal server is down.");
- }
- }
-
- });
- } catch (Throwable e) {
- logger.error("## Something goes wrong when starting up the canal Server:", e);
- System.exit(0);
- }
- }
-
- private static void setGlobalUncaughtExceptionHandler() {
- Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- logger.error("UnCaughtException", e);
- }
- });
+ CanalKafkaStarter.init();
}
}
\ No newline at end of file
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java
new file mode 100644
index 00000000..0b033716
--- /dev/null
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java
@@ -0,0 +1,76 @@
+package com.alibaba.otter.canal.kafka;
+
+import com.alibaba.otter.canal.deployer.CanalController;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.util.Properties;
+
+/**
+ * canal server 启动类
+ *
+ * @author machengyuan 2018-6-11 下午05:30:49
+ * @version 1.0.0
+ */
+public class CanalServerStarter {
+ private static final String CLASSPATH_URL_PREFIX = "classpath:";
+ private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class);
+
+ private static boolean running = false;
+
+ public static void init() {
+ try {
+ logger.info("## set default uncaught exception handler");
+ setGlobalUncaughtExceptionHandler();
+
+ logger.info("## load canal configurations");
+ String conf = System.getProperty("canal.conf", "classpath:canal.properties");
+ Properties properties = new Properties();
+ if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
+ conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
+ properties.load(com.alibaba.otter.canal.deployer.CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
+ } else {
+ properties.load(new FileInputStream(conf));
+ }
+
+ logger.info("## start the canal server.");
+ final CanalController controller = new CanalController(properties);
+ controller.start();
+ running = true;
+ logger.info("## the canal server is running now ......");
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+
+ public void run() {
+ try {
+ logger.info("## stop the canal server");
+ running = false;
+ controller.stop();
+ } catch (Throwable e) {
+ logger.warn("##something goes wrong when stopping canal Server:", e);
+ } finally {
+ logger.info("## canal server is down.");
+ }
+ }
+
+ });
+ } catch (Throwable e) {
+ logger.error("## Something goes wrong when starting up the canal Server:", e);
+ System.exit(0);
+ }
+ }
+
+ public static boolean isRunning(){
+ return running;
+ }
+
+ private static void setGlobalUncaughtExceptionHandler() {
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ logger.error("UnCaughtException", e);
+ }
+ });
+ }
+}
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
index 9a64b1a5..ae094b27 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java
@@ -1,6 +1,7 @@
package com.alibaba.otter.canal.kafka.producer;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic;
+import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
@@ -9,6 +10,7 @@ import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
import java.util.Properties;
/**
@@ -22,7 +24,7 @@ public class CanalKafkaProducer {
private static Producer producer;
- public static void init(KafkaProperties kafkaProperties) {
+ static void init(KafkaProperties kafkaProperties) {
Properties properties = new Properties();
properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", "all");
@@ -33,24 +35,45 @@ public class CanalKafkaProducer {
properties.put("key.serializer", StringSerializer.class.getName());
properties.put("value.serializer", MessageSerializer.class.getName());
producer = new KafkaProducer(properties);
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
-
- public void run() {
- try {
- logger.info("## stop the kafka producer");
- producer.close();
- } catch (Throwable e) {
- logger.warn("##something goes wrong when stopping kafka producer:", e);
- } finally {
- logger.info("## kafka producer is down.");
- }
- }
-
- });
}
- public static void send(Topic topic, Message message) {
+ static void stop() {
+ try {
+ logger.info("## stop the kafka producer");
+ producer.close();
+ } catch (Throwable e) {
+ logger.warn("##something goes wrong when stopping kafka producer:", e);
+ } finally {
+ logger.info("## kafka producer is down.");
+ }
+ }
+
+ static void send(Topic topic, Message message) {
+ try {
+ List entries = message.getEntries();
+ boolean flag = false;
+ if (!entries.isEmpty()) {
+ for (CanalEntry.Entry entry : entries) {
+ CanalEntry.RowChange rowChage = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+ if (rowChage.getIsDdl()) {
+ flag = true;
+ break;
+ } else {
+ if (!rowChage.getRowDatasList().isEmpty()) {
+ flag = true;
+ break;
+ }
+ }
+ }
+ }
+ if (!flag) {
+ return;
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return;
+ }
+
ProducerRecord record;
if (topic.getPartition() != null) {
record = new ProducerRecord(topic.getTopic(), topic.getPartition(), null, message);
diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
index bc048956..a0dc3691 100644
--- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
+++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java
@@ -1,60 +1,106 @@
package com.alibaba.otter.canal.kafka.producer;
+import java.io.FileInputStream;
import java.util.List;
+import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import com.alibaba.otter.canal.kafka.CanalServerStarter;
import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic;
+import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+/**
+ * kafka 启动类
+ *
+ * @author machengyuan 2018-6-11 下午05:30:49
+ * @version 1.0.0
+ */
public class CanalKafkaStarter {
+ private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaStarter.class);
private volatile static boolean running = false;
- private static short CLIENT_ID = 1001;
- public static void init(KafkaProperties kafkaProperties) {
- //初始化 kafka producer
- CanalKafkaProducer.init(kafkaProperties);
+ private static ExecutorService executorService;
- //对应每个instance启动一个worker线程
- List topics = kafkaProperties.getTopics();
-
- ExecutorService executorService = Executors.newFixedThreadPool(topics.size());
-
- for (final Topic topic : topics) {
- executorService.execute(new Runnable() {
- @Override
- public void run() {
- worker(topic);
- }
- });
- }
- running = true;
- }
-
- private static void sleep(long time) {
+ public static void init() {
try {
- Thread.sleep(time);
- } catch (InterruptedException e) {
- //ignore
+
+ logger.info("## load kafka configurations");
+ String conf = System.getProperty("canal.conf", "classpath:kafka.yml");
+
+ KafkaProperties kafkaProperties;
+ if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
+ conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
+ kafkaProperties = new Yaml().loadAs(CanalKafkaStarter.class.getClassLoader().getResourceAsStream(conf), KafkaProperties.class);
+ } else {
+ kafkaProperties = new Yaml().loadAs(new FileInputStream(conf), KafkaProperties.class);
+ }
+
+ //初始化 kafka producer
+ CanalKafkaProducer.init(kafkaProperties);
+
+ //对应每个instance启动一个worker线程
+ List topics = kafkaProperties.getTopics();
+
+ executorService = Executors.newFixedThreadPool(topics.size());
+
+ logger.info("## start the kafka workers.");
+ for (final Topic topic : topics) {
+ executorService.execute(new Runnable() {
+ @Override
+ public void run() {
+ worker(topic);
+ }
+ });
+ }
+ running = true;
+ logger.info("## the kafka workers is running now ......");
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ public void run() {
+ try {
+ logger.info("## stop the kafka workers");
+ running = false;
+ executorService.shutdown();
+ CanalKafkaProducer.stop();
+ } catch (Throwable e) {
+ logger.warn("##something goes wrong when stopping kafka workers:", e);
+ } finally {
+ logger.info("## canal kafka is down.");
+ }
+ }
+
+ });
+
+ } catch (Throwable e) {
+ logger.error("## Something goes wrong when starting up the canal kafka workers:", e);
+ System.exit(0);
}
}
- public static void worker(Topic topic) {
+
+ private static void worker(Topic topic) {
while (!running) ;
- //TODO 等待canal server启动完毕
+ while (!CanalServerStarter.isRunning()) ; //等待server启动完成
logger.info("## start the canal consumer: {}.", topic.getDestination());
CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
- ClientIdentity clientIdentity = new ClientIdentity(topic.getDestination(), CLIENT_ID, "");
+ ClientIdentity clientIdentity = new ClientIdentity(topic.getDestination(), (short) 1001, "");
while (running) {
try {
if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
- sleep(3000);
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ //ignore
+ }
continue;
}
server.subscribe(clientIdentity);
@@ -66,7 +112,11 @@ public class CanalKafkaStarter {
try {
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
- sleep(1000);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ //ignore
+ }
} else {
CanalKafkaProducer.send(topic, message);
}
@@ -77,7 +127,6 @@ public class CanalKafkaStarter {
} catch (Exception e) {
server.rollback(clientIdentity);
logger.error(e.getMessage(), e);
- sleep(1000);
}
}
} catch (Exception e) {
diff --git a/kafka/src/main/resources/kafka.yml b/kafka/src/main/resources/kafka.yml
index 2d07cf14..af84977d 100644
--- a/kafka/src/main/resources/kafka.yml
+++ b/kafka/src/main/resources/kafka.yml
@@ -1,12 +1,12 @@
-servers: localhost:6667
+servers: slave1.test.apitops.com:6667,slave2.test.apitops.com:6667,slave3.test.apitops.com:6667
retries: 0
batchSize: 16384
lingerMs: 1
bufferMemory: 33554432
topics:
- - topic: exampe
+ - topic: example2
partition:
- destination: example
+ destination: example2
diff --git a/kafka/src/main/resources/logback.xml b/kafka/src/main/resources/logback.xml
new file mode 100644
index 00000000..158c1699
--- /dev/null
+++ b/kafka/src/main/resources/logback.xml
@@ -0,0 +1,85 @@
+
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+ destination
+ canal
+
+
+
+ ../logs/${destination}/${destination}.log
+
+
+ ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 512MB
+
+ 60
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+
+
+
+
+
+
+
+
+ destination
+ canal
+
+
+
+ ../logs/${destination}/meta.log
+
+
+ ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz
+
+
+ 32MB
+
+ 60
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file