Add RocketMQ canal connector

This commit is contained in:
duhengforever
2018-09-20 15:48:52 +08:00
parent 00b5ade787
commit d5aa324ce4
24 changed files with 881 additions and 328 deletions

View File

@@ -1,24 +1,26 @@
package com.alibaba.otter.canal.deployer;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
import com.alibaba.otter.canal.server.CanalMQStarter;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import java.io.FileInputStream;
import java.util.Properties;
import com.alibaba.otter.canal.kafka.CanalKafkaStarter;
import com.alibaba.otter.canal.server.CanalServerStarter;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* canal独立版本启动的入口类
*
*
* @author jianghang 2012-11-6 下午05:20:49
* @version 1.0.0
*/
public class CanalLauncher {
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class);
public static void main(String[] args) throws Throwable {
try {
@@ -54,15 +56,18 @@ public class CanalLauncher {
});
CanalServerStarter canalServerStarter = null;
CanalMQProducer canalMQProducer = null;
String serverMode = properties.getProperty(CanalConstants.CANAL_SERVER_MODE, "tcp");
if (serverMode.equalsIgnoreCase("kafka")) {
canalServerStarter = new CanalKafkaStarter();
} else if (serverMode.equalsIgnoreCase("rocketMQ")) {
// 预留rocketMQ启动
canalMQProducer = new CanalKafkaProducer();
} else if (serverMode.equalsIgnoreCase("rocketmq")) {
canalMQProducer = new CanalRocketMQProducer();
}
if (canalServerStarter != null) {
canalServerStarter.init();
if (canalMQProducer != null) {
CanalMQStarter canalServerStarter = new CanalMQStarter(canalMQProducer);
if (canalServerStarter != null) {
canalServerStarter.init();
}
}
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the canal Server:", e);

View File

@@ -9,8 +9,8 @@ canal.zkServers=
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, rocketMQ
canal.serverMode = tcp
# tcp, kafka, RocketMQ
canal.serverMode = rocketmq
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000

View File

@@ -6,7 +6,7 @@
canal.instance.gtidon=false
# position info
canal.instance.master.address=127.0.0.1:3306
canal.instance.master.address=120.78.225.173:3307
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
@@ -18,8 +18,9 @@ canal.instance.rds.secretkey=
canal.instance.rds.instanceId=
# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://120.78.225.173:3306/test
#canal.instance.mysql.slaveId = 1234
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal
@@ -31,7 +32,7 @@ canal.instance.tsdb.enable=true
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.dbPassword=P@ssw0rd
canal.instance.connectionCharset=UTF-8
# table regex

View File

@@ -1,4 +1,4 @@
servers: slave1:6667,slave2:6667,slave3:6667
servers: localhost:9876 #for rocketmq: means the nameserver
retries: 0
batchSize: 16384
lingerMs: 1