mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
模块化mq connector (#2562)
* add connector module, add kafka producer connector module for server * add rocketMQ producer connector module for server * add rabbitMQ producer connector module for server * optimize code, add annotation * add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
296
deployer/pom.xml
296
deployer/pom.xml
@@ -1,127 +1,185 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.1.5-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.deployer</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal deployer module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.server</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.1.5-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.deployer</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal deployer module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.server</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 这里指定runtime的metrics provider-->
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.prometheus</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 -->
|
||||
<plugin>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<configuration>
|
||||
<archive>
|
||||
<addMavenDescriptor>true</addMavenDescriptor>
|
||||
</archive>
|
||||
<excludes>
|
||||
<exclude>**/logback.xml</exclude>
|
||||
<exclude>**/canal.properties</exclude>
|
||||
<exclude>**/spring/**</exclude>
|
||||
<exclude>**/example/**</exclude>
|
||||
<exclude>**/mq.yml</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<!-- 这里指定runtime的metrics provider-->
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.prometheus</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<!-- 这是最新版本，推荐使用这个版本 -->
|
||||
<version>2.2.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>assemble</id>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<attach>false</attach>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.kafka</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>*</artifactId>
|
||||
<groupId>*</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<classifier>jar-with-dependencies</classifier>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.rocketmq</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>*</artifactId>
|
||||
<groupId>*</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<classifier>jar-with-dependencies</classifier>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.rabbitmq</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>*</artifactId>
|
||||
<groupId>*</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<classifier>jar-with-dependencies</classifier>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!release</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<configuration>
|
||||
<archive>
|
||||
<addMavenDescriptor>true</addMavenDescriptor>
|
||||
</archive>
|
||||
<excludes>
|
||||
<exclude>**/logback.xml</exclude>
|
||||
<exclude>**/canal.properties</exclude>
|
||||
<exclude>**/spring/**</exclude>
|
||||
<exclude>**/example/**</exclude>
|
||||
<exclude>**/mq.yml</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
|
||||
<descriptors>
|
||||
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
|
||||
</descriptors>
|
||||
<finalName>canal</finalName>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<!-- 这是最新版本，推荐使用这个版本 -->
|
||||
<version>2.2.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>assemble</id>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<attach>false</attach>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.10</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies-to-canal-client-service</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<includeClassifiers>jar-with-dependencies</includeClassifiers>
|
||||
<outputDirectory>${project.basedir}/target/canal/plugin</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</profile>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!release</value>
|
||||
</property>
|
||||
</activation>
|
||||
|
||||
<profile>
|
||||
<id>release</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>release</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
|
||||
<descriptors>
|
||||
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
|
||||
</descriptors>
|
||||
<finalName>canal</finalName>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- 发布模式使用的maven assembly插件描述文件 -->
|
||||
<descriptors>
|
||||
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
|
||||
</descriptors>
|
||||
<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
|
||||
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
|
||||
<outputDirectory>${project.parent.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>release</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>release</value>
|
||||
</property>
|
||||
</activation>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- 发布模式使用的maven assembly插件描述文件 -->
|
||||
<descriptors>
|
||||
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
|
||||
</descriptors>
|
||||
<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
|
||||
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
|
||||
<outputDirectory>${project.parent.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
@@ -42,6 +42,10 @@
|
||||
<exclude>**/*</exclude>
|
||||
</excludes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}/target/canal/plugin</directory>
|
||||
<outputDirectory>/plugin/</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
|
||||
@@ -46,36 +46,37 @@ public class CanalConstants {
|
||||
|
||||
public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";
|
||||
|
||||
public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
|
||||
public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
|
||||
public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
|
||||
public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
|
||||
public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
|
||||
public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
|
||||
public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
|
||||
public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
|
||||
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
|
||||
public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
|
||||
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
|
||||
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
|
||||
public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
|
||||
public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
|
||||
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
|
||||
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
|
||||
public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
|
||||
public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
|
||||
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
|
||||
public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
|
||||
public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
|
||||
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
|
||||
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
|
||||
public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
|
||||
public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
|
||||
public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
|
||||
public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
|
||||
public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
|
||||
public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
|
||||
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
|
||||
|
||||
// public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
|
||||
// public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
|
||||
// public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
|
||||
// public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
|
||||
// public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
|
||||
// public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
|
||||
// public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
|
||||
// public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
|
||||
// public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
|
||||
// public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
|
||||
// public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
|
||||
// public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
|
||||
// public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
|
||||
// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
|
||||
// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
|
||||
// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
|
||||
// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
|
||||
// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
|
||||
// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
|
||||
// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
|
||||
// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
|
||||
// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
|
||||
// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
|
||||
// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
|
||||
// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
|
||||
// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
|
||||
// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
|
||||
// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
|
||||
|
||||
public static String getInstanceModeKey(String destination) {
|
||||
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
|
||||
|
||||
@@ -2,35 +2,36 @@ package com.alibaba.otter.canal.deployer;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import com.alibaba.otter.canal.connector.core.config.MQProperties;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
|
||||
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
|
||||
import com.alibaba.otter.canal.deployer.admin.CanalAdminController;
|
||||
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
|
||||
import com.alibaba.otter.canal.rabbitmq.CanalRabbitMQProducer;
|
||||
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
|
||||
import com.alibaba.otter.canal.server.CanalMQStarter;
|
||||
import com.alibaba.otter.canal.spi.CanalMQProducer;
|
||||
|
||||
/**
|
||||
* Canal server 启动类
|
||||
*
|
||||
* @author rewerma 2018-12-30 下午05:12:16
|
||||
* @version 1.0.1
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.2
|
||||
*/
|
||||
public class CanalStarter {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
|
||||
|
||||
private CanalController controller = null;
|
||||
private CanalMQProducer canalMQProducer = null;
|
||||
private Thread shutdownThread = null;
|
||||
private CanalMQStarter canalMQStarter = null;
|
||||
private static final String CONNECTOR_SPI_DIR = "/plugin";
|
||||
private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";
|
||||
|
||||
private CanalController controller = null;
|
||||
private CanalMQProducer canalMQProducer = null;
|
||||
private Thread shutdownThread = null;
|
||||
private CanalMQStarter canalMQStarter = null;
|
||||
private volatile Properties properties;
|
||||
private volatile boolean running = false;
|
||||
private volatile boolean running = false;
|
||||
|
||||
private CanalAdminWithNetty canalAdmin;
|
||||
|
||||
@@ -61,20 +62,23 @@ public class CanalStarter {
|
||||
*/
|
||||
public synchronized void start() throws Throwable {
|
||||
String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
|
||||
if (serverMode.equalsIgnoreCase("kafka")) {
|
||||
canalMQProducer = new CanalKafkaProducer();
|
||||
} else if (serverMode.equalsIgnoreCase("rocketmq")) {
|
||||
canalMQProducer = new CanalRocketMQProducer();
|
||||
} else if (serverMode.equalsIgnoreCase("rabbitmq")) {
|
||||
canalMQProducer = new CanalRabbitMQProducer();
|
||||
if (!"tcp".equalsIgnoreCase(serverMode)) {
|
||||
ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
|
||||
canalMQProducer = loader
|
||||
.getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
|
||||
if (canalMQProducer != null) {
|
||||
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
||||
Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
|
||||
canalMQProducer.init(properties);
|
||||
Thread.currentThread().setContextClassLoader(cl);
|
||||
}
|
||||
}
|
||||
|
||||
MQProperties mqProperties = null;
|
||||
if (canalMQProducer != null) {
|
||||
mqProperties = buildMQProperties(properties);
|
||||
MQProperties mqProperties = canalMQProducer.getMqProperties();
|
||||
// disable netty
|
||||
System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
|
||||
if (mqProperties.getFlatMessage()) {
|
||||
if (mqProperties.isFlatMessage()) {
|
||||
// 设置为raw避免ByteString->Entry的二次解析
|
||||
System.setProperty("canal.instance.memory.rawEntry", "false");
|
||||
}
|
||||
@@ -104,7 +108,7 @@ public class CanalStarter {
|
||||
if (canalMQProducer != null) {
|
||||
canalMQStarter = new CanalMQStarter(canalMQProducer);
|
||||
String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
|
||||
canalMQStarter.start(mqProperties, destinations);
|
||||
canalMQStarter.start(destinations);
|
||||
controller.setCanalMQStarter(canalMQStarter);
|
||||
}
|
||||
|
||||
@@ -119,11 +123,15 @@ public class CanalStarter {
|
||||
|
||||
String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP);
|
||||
|
||||
logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", port, user, passwd, ip);
|
||||
logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}",
|
||||
port,
|
||||
user,
|
||||
passwd,
|
||||
ip);
|
||||
|
||||
CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
|
||||
canalAdminWithNetty.setCanalAdmin(canalAdmin);
|
||||
canalAdminWithNetty.setPort(Integer.valueOf(port));
|
||||
canalAdminWithNetty.setPort(Integer.parseInt(port));
|
||||
canalAdminWithNetty.setIp(ip);
|
||||
canalAdminWithNetty.start();
|
||||
this.canalAdmin = canalAdminWithNetty;
|
||||
@@ -162,156 +170,4 @@ public class CanalStarter {
|
||||
}
|
||||
running = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造MQ对应的配置
|
||||
*
|
||||
* @param properties canal.properties 配置
|
||||
* @return
|
||||
*/
|
||||
private static MQProperties buildMQProperties(Properties properties) {
|
||||
MQProperties mqProperties = new MQProperties();
|
||||
String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
|
||||
if (!StringUtils.isEmpty(servers)) {
|
||||
mqProperties.setServers(servers);
|
||||
}
|
||||
String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
|
||||
if (!StringUtils.isEmpty(retires)) {
|
||||
mqProperties.setRetries(Integer.valueOf(retires));
|
||||
}
|
||||
String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
|
||||
if (!StringUtils.isEmpty(batchSize)) {
|
||||
mqProperties.setBatchSize(Integer.valueOf(batchSize));
|
||||
}
|
||||
String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
|
||||
if (!StringUtils.isEmpty(lingerMs)) {
|
||||
mqProperties.setLingerMs(Integer.valueOf(lingerMs));
|
||||
}
|
||||
String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
|
||||
if (!StringUtils.isEmpty(maxRequestSize)) {
|
||||
mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
|
||||
}
|
||||
String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
|
||||
if (!StringUtils.isEmpty(bufferMemory)) {
|
||||
mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
|
||||
}
|
||||
String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
|
||||
if (!StringUtils.isEmpty(canalBatchSize)) {
|
||||
mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
|
||||
}
|
||||
String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
|
||||
if (!StringUtils.isEmpty(canalGetTimeout)) {
|
||||
mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
|
||||
}
|
||||
String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
|
||||
if (!StringUtils.isEmpty(flatMessage)) {
|
||||
mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
|
||||
}
|
||||
String parallelThreadSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PARALLELTHREADSIZE);
|
||||
if (!StringUtils.isEmpty(parallelThreadSize)) {
|
||||
mqProperties.setParallelThreadSize(Integer.valueOf(parallelThreadSize));
|
||||
}
|
||||
String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
|
||||
if (!StringUtils.isEmpty(compressionType)) {
|
||||
mqProperties.setCompressionType(compressionType);
|
||||
}
|
||||
String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
|
||||
if (!StringUtils.isEmpty(acks)) {
|
||||
mqProperties.setAcks(acks);
|
||||
}
|
||||
String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
|
||||
if (!StringUtils.isEmpty(aliyunAccessKey)) {
|
||||
mqProperties.setAliyunAccessKey(aliyunAccessKey);
|
||||
}
|
||||
String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
|
||||
if (!StringUtils.isEmpty(aliyunSecretKey)) {
|
||||
mqProperties.setAliyunSecretKey(aliyunSecretKey);
|
||||
}
|
||||
|
||||
String producerGroup = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PRODUCERGROUP);
|
||||
if (!StringUtils.isEmpty(producerGroup)) {
|
||||
mqProperties.setProducerGroup(producerGroup);
|
||||
}
|
||||
|
||||
String enableMessageTrace = CanalController.getProperty(properties,
|
||||
CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE);
|
||||
if (!StringUtils.isEmpty(enableMessageTrace)) {
|
||||
mqProperties.setEnableMessageTrace(Boolean.valueOf(enableMessageTrace));
|
||||
}
|
||||
|
||||
String accessChannel = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
|
||||
if (!StringUtils.isEmpty(accessChannel)) {
|
||||
mqProperties.setAccessChannel(accessChannel);
|
||||
}
|
||||
|
||||
String customizedTraceTopic = CanalController.getProperty(properties,
|
||||
CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC);
|
||||
if (!StringUtils.isEmpty(customizedTraceTopic)) {
|
||||
mqProperties.setCustomizedTraceTopic(customizedTraceTopic);
|
||||
}
|
||||
|
||||
String namespace = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_NAMESPACE);
|
||||
if (!StringUtils.isEmpty(namespace)) {
|
||||
mqProperties.setNamespace(namespace);
|
||||
}
|
||||
|
||||
String kafkaKerberosEnable = CanalController.getProperty(properties,
|
||||
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
|
||||
if (!StringUtils.isEmpty(kafkaKerberosEnable)) {
|
||||
mqProperties.setKerberosEnable(Boolean.valueOf(kafkaKerberosEnable));
|
||||
}
|
||||
|
||||
String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties,
|
||||
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH);
|
||||
if (!StringUtils.isEmpty(kafkaKerberosKrb5Filepath)) {
|
||||
mqProperties.setKerberosKrb5FilePath(kafkaKerberosKrb5Filepath);
|
||||
}
|
||||
|
||||
String kafkaKerberosJaasFilepath = CanalController.getProperty(properties,
|
||||
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH);
|
||||
if (!StringUtils.isEmpty(kafkaKerberosJaasFilepath)) {
|
||||
mqProperties.setKerberosJaasFilePath(kafkaKerberosJaasFilepath);
|
||||
}
|
||||
|
||||
String vhost = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_VHOST);
|
||||
if (!StringUtils.isEmpty(vhost)) {
|
||||
mqProperties.setVhost(vhost);
|
||||
}
|
||||
|
||||
String username = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_USERNAME);
|
||||
if (!StringUtils.isEmpty(username)) {
|
||||
mqProperties.setUsername(username);
|
||||
}
|
||||
|
||||
String password = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PASSWORD);
|
||||
if (!StringUtils.isEmpty(password)) {
|
||||
mqProperties.setPassword(password);
|
||||
}
|
||||
|
||||
String aliyunUID = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ALIYUN_UID);
|
||||
if (!StringUtils.isEmpty(aliyunUID)) {
|
||||
mqProperties.setAliyunUID(Long.valueOf(aliyunUID));
|
||||
}
|
||||
|
||||
String exchange = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_EXCHANGE);
|
||||
if (!StringUtils.isEmpty(exchange)) {
|
||||
mqProperties.setExchange(exchange);
|
||||
}
|
||||
|
||||
String databaseHash = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_DATABASE_HASH);
|
||||
if (!StringUtils.isEmpty(databaseHash)){
|
||||
mqProperties.setDatabaseHash(Boolean.valueOf(databaseHash));
|
||||
}
|
||||
|
||||
for (Object key : properties.keySet()) {
|
||||
key = StringUtils.trim(key.toString());
|
||||
if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) {
|
||||
String value = CanalController.getProperty(properties, (String) key);
|
||||
String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1);
|
||||
mqProperties.getProperties().put(subKey, value);
|
||||
}
|
||||
}
|
||||
|
||||
return mqProperties;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ canal.zkServers =
|
||||
# flush data to zk
|
||||
canal.zookeeper.flush.period = 1000
|
||||
canal.withoutNetty = false
|
||||
# tcp, kafka, RocketMQ
|
||||
# tcp, kafka, rocketMQ, rabbitMQ
|
||||
canal.serverMode = tcp
|
||||
# flush meta cursor/parse position to file
|
||||
canal.file.data.dir = ${canal.conf.dir}
|
||||
@@ -86,10 +86,6 @@ canal.instance.tsdb.snapshot.interval = 24
|
||||
# purge snapshot expire , default 360 hour(15 days)
|
||||
canal.instance.tsdb.snapshot.expire = 360
|
||||
|
||||
# aliyun ak/sk , support rds/mq
|
||||
canal.aliyun.accessKey =
|
||||
canal.aliyun.secretKey =
|
||||
|
||||
#################################################
|
||||
######### destinations #############
|
||||
#################################################
|
||||
@@ -111,35 +107,54 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
|
||||
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
|
||||
|
||||
##################################################
|
||||
######### MQ #############
|
||||
######### MQ Properties #############
|
||||
##################################################
|
||||
canal.mq.servers = 127.0.0.1:6667
|
||||
canal.mq.retries = 0
|
||||
canal.mq.batchSize = 16384
|
||||
canal.mq.maxRequestSize = 1048576
|
||||
canal.mq.lingerMs = 100
|
||||
canal.mq.bufferMemory = 33554432
|
||||
canal.mq.canalBatchSize = 50
|
||||
canal.mq.canalGetTimeout = 100
|
||||
canal.mq.parallelThreadSize = 8
|
||||
canal.mq.flatMessage = true
|
||||
canal.mq.compressionType = none
|
||||
canal.mq.acks = all
|
||||
#canal.mq.properties. =
|
||||
canal.mq.producerGroup = test
|
||||
# Set this value to "cloud", if you want open message trace feature in aliyun.
|
||||
canal.mq.accessChannel = local
|
||||
# aliyun mq namespace
|
||||
#canal.mq.namespace =
|
||||
canal.mq.vhost=
|
||||
canal.mq.exchange=
|
||||
canal.mq.username=
|
||||
canal.mq.password=
|
||||
canal.mq.aliyunuid=
|
||||
canal.mq.flat.message = true
|
||||
canal.mq.database.hash = true
|
||||
canal.mq.parallel.thread.size = 8
|
||||
canal.mq.canal.batch.size = 50
|
||||
canal.mq.canal.fetch.timeout = 100
|
||||
# Set this value to "cloud", if you want open message trace feature in aliyun.
|
||||
canal.mq.access.channel = local
|
||||
|
||||
# aliyun ak/sk , support rds/mq
|
||||
canal.aliyun.accessKey =
|
||||
canal.aliyun.secretKey =
|
||||
canal.aliyun.uid=
|
||||
|
||||
##################################################
|
||||
######### Kafka Kerberos Info #############
|
||||
######### Kafka #############
|
||||
##################################################
|
||||
kafka.bootstrap.servers = 127.0.0.1:9092
|
||||
kafka.acks = all
|
||||
kafka.compression.type = none
|
||||
kafka.batch.size = 16384
|
||||
kafka.linger.ms = 1
|
||||
kafka.max.request.size = 1048576
|
||||
kafka.buffer.memory = 33554432
|
||||
kafka.max.in.flight.requests.per.connection = 1
|
||||
kafka.retries = 0
|
||||
|
||||
canal.mq.kafka.kerberos.enable = false
|
||||
canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
|
||||
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
|
||||
canal.mq.kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
|
||||
canal.mq.kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
|
||||
|
||||
##################################################
|
||||
######### RocketMQ #############
|
||||
##################################################
|
||||
rocketmq.producer.group = test
|
||||
rocketmq.enable.message.trace = false
|
||||
rocketmq.customized.trace.topic =
|
||||
rocketmq.namespace =
|
||||
rocketmq.namesrv.addr = 127.0.0.1:9876
|
||||
rocketmq.retry.times.when.send.failed = 0
|
||||
rocketmq.vip.channel.enabled = false
|
||||
|
||||
##################################################
|
||||
######### RabbitMQ #############
|
||||
##################################################
|
||||
rabbitmq.host =
|
||||
rabbitmq.virtual.host =
|
||||
rabbitmq.exchange =
|
||||
rabbitmq.username =
|
||||
rabbitmq.password =
|
||||
@@ -73,11 +73,15 @@
|
||||
<level value="INFO" />
|
||||
<appender-ref ref="CANAL-META" />
|
||||
</logger>
|
||||
<logger name="com.alibaba.otter.canal.kafka" additivity="false">
|
||||
<logger name="com.alibaba.otter.canal.connector.kafka" additivity="false">
|
||||
<level value="INFO" />
|
||||
<appender-ref ref="CANAL-ROOT" />
|
||||
</logger>
|
||||
<logger name="com.alibaba.otter.canal.rabbitmq" additivity="false">
|
||||
<logger name="com.alibaba.otter.canal.connector.rocketmq" additivity="false">
|
||||
<level value="INFO" />
|
||||
<appender-ref ref="CANAL-ROOT" />
|
||||
</logger>
|
||||
<logger name="com.alibaba.otter.canal.connector.rabbitmq" additivity="false">
|
||||
<level value="INFO" />
|
||||
<appender-ref ref="CANAL-ROOT" />
|
||||
</logger>
|
||||
|
||||
Reference in New Issue
Block a user