mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
模块化mq connector (#2562)
* add connector module, add kafka producer connector module for server * add rocketMQ producer connector module for server * add rabbitMQ producer connector module for server * optimize code, add annotation * add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
124
connector/tcp-connector/pom.xml
Normal file
124
connector/tcp-connector/pom.xml
Normal file
@@ -0,0 +1,124 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>canal.connector</artifactId>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<version>1.1.5-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.tcp</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal connector tcp module for otter ${project.version}</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.core</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-orm</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-aop</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-client</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.aliyun.openservices</groupId>
|
||||
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4</version>
|
||||
<configuration>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>1.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<tasks>
|
||||
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin"
|
||||
overwrite="true">
|
||||
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
|
||||
<include name="connector.tcp-${project.version}-jar-with-dependencies.jar"/>
|
||||
</fileset>
|
||||
</copy>
|
||||
</tasks>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.alibaba.otter.canal.connector.tcp.config;
|
||||
|
||||
public class TCPConstants {
|
||||
|
||||
public static final String ROOT = "canal";
|
||||
|
||||
public static final String CANAL_TCP_HOST = ROOT + "." + "tcp.server.host";
|
||||
public static final String CANAL_TCP_ZK_HOSTS = ROOT + "." + "tcp.zookeeper.hosts";
|
||||
public static final String CANAL_TCP_USERNAME = ROOT + "." + "tcp.username";
|
||||
public static final String CANAL_TCP_PASSWORD = ROOT + "." + "tcp.password";
|
||||
public static final String CANAL_TCP_BATCH_SIZE = ROOT + "." + "tcp.batch.size";
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package com.alibaba.otter.canal.connector.tcp.consumer;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
|
||||
import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
|
||||
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
|
||||
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
|
||||
import com.alibaba.otter.canal.connector.core.spi.SPI;
|
||||
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
|
||||
import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* TCP 消费者连接器, 一个destination对应一个SPI实例
|
||||
*
|
||||
* @author rewerma 2020-01-30
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SPI("tcp")
|
||||
public class CanalTCPConsumer implements CanalMsgConsumer {
|
||||
|
||||
private static final String PREFIX_TCP_CONFIG = "canal.tcp.";
|
||||
|
||||
private Long currentBatchId = null;
|
||||
|
||||
private CanalConnector canalConnector;
|
||||
|
||||
private int batchSize = 500;
|
||||
|
||||
@Override
|
||||
public void init(Properties properties, String destination, String groupId) {
|
||||
// load config
|
||||
String host = properties.getProperty(TCPConstants.CANAL_TCP_HOST);
|
||||
String username = properties.getProperty(TCPConstants.CANAL_TCP_USERNAME);
|
||||
String password = properties.getProperty(TCPConstants.CANAL_TCP_PASSWORD);
|
||||
String zkHosts = properties.getProperty(TCPConstants.CANAL_TCP_ZK_HOSTS);
|
||||
String batchSizePro = properties.getProperty(TCPConstants.CANAL_TCP_BATCH_SIZE);
|
||||
if (batchSizePro != null) {
|
||||
batchSize = Integer.parseInt(batchSizePro);
|
||||
}
|
||||
if (host != null) {
|
||||
String[] ipPort = host.split(":");
|
||||
SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
|
||||
this.canalConnector = new SimpleCanalConnector(sa, username, password, destination);
|
||||
} else {
|
||||
this.canalConnector = new ClusterCanalConnector(username,
|
||||
password,
|
||||
destination,
|
||||
new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkHosts)));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void connect() {
|
||||
canalConnector.connect();
|
||||
canalConnector.subscribe();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
|
||||
try {
|
||||
Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
|
||||
long batchId = message.getId();
|
||||
currentBatchId = batchId;
|
||||
int size = message.getEntries().size();
|
||||
if (batchId == -1 || size == 0) {
|
||||
return null;
|
||||
} else {
|
||||
return MessageUtil.convert(message);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rollback() {
|
||||
if (currentBatchId != null && currentBatchId != -1) {
|
||||
canalConnector.rollback(currentBatchId);
|
||||
currentBatchId = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ack() {
|
||||
if (currentBatchId != null) {
|
||||
canalConnector.ack(currentBatchId);
|
||||
currentBatchId = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void disconnect() {
|
||||
canalConnector.unsubscribe();
|
||||
canalConnector.disconnect();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
tcp=com.alibaba.otter.canal.connector.tcp.consumer.CanalTCPConsumer
|
||||
Reference in New Issue
Block a user