chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

模块化mq connector (#2562)

Browse Source 
* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
2020-02-28 11:00:08 +08:00
committed by GitHub
parent 4fc7e5f7d9
commit 265a8e8594
82 changed files with 4932 additions and 2412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
client-adapter/common/pom.xml
View File

@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent> <parent>
<artifactId>canal.client-adapter</artifactId> <artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
@@ -17,6 +18,11 @@
<artifactId>canal.protocol</artifactId> <artifactId>canal.protocol</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.springframework</groupId> <groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId> <artifactId>spring-core</artifactId>

13
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties;
/** /**
* 配置信息类 * 配置信息类
@@ -27,7 +28,7 @@ public class CanalClientConfig {
// 重试次数 // 重试次数
private Integer retries; private Integer retries;
// 消费超时时间 // 消费超时时间
private Long timeout; private Long timeout = 500L;
// 模式 tcp kafka rocketMQ // 模式 tcp kafka rocketMQ
private String mode = "tcp"; private String mode = "tcp";
// aliyun ak/sk // aliyun ak/sk
@@ -39,6 +40,8 @@ public class CanalClientConfig {
// rabbitmq vhost // rabbitmq vhost
private String vhost = "/"; private String vhost = "/";
private Properties consumerProperties;
private Long resourceOwnerId; private Long resourceOwnerId;
// 是否启用消息轨迹 // 是否启用消息轨迹
private boolean enableMessageTrace; private boolean enableMessageTrace;
@@ -163,6 +166,14 @@ public class CanalClientConfig {
this.vhost = vhost; this.vhost = vhost;
} }
public Properties getConsumerProperties() {
return consumerProperties;
}
public void setConsumerProperties(Properties consumerProperties) {
this.consumerProperties = consumerProperties;
}
public Long getResourceOwnerId() { public Long getResourceOwnerId() {
return resourceOwnerId; return resourceOwnerId;
} }

40
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.*; import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
@@ -131,10 +132,13 @@ public class MessageUtil {
return dmls; return dmls;
} }
public static List<Dml> flatMessage2Dml(String destination, String groupId, List<FlatMessage> flatMessages) { public static List<Dml> flatMessage2Dml(String destination, String groupId, List<CommonMessage> commonMessages) {
List<Dml> dmls = new ArrayList<Dml>(flatMessages.size()); if (commonMessages == null) {
for (FlatMessage flatMessage : flatMessages) { return new ArrayList<>();
Dml dml = flatMessage2Dml(destination, groupId, flatMessage); }
List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
for (CommonMessage commonMessage : commonMessages) {
Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
if (dml != null) { if (dml != null) {
dmls.add(dml); dmls.add(dml);
} }
@@ -143,31 +147,31 @@ public class MessageUtil {
return dmls; return dmls;
} }
public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) { public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) {
if (flatMessage == null) { if (commonMessage == null) {
return null; return null;
} }
Dml dml = new Dml(); Dml dml = new Dml();
dml.setDestination(destination); dml.setDestination(destination);
dml.setGroupId(groupId); dml.setGroupId(groupId);
dml.setDatabase(flatMessage.getDatabase()); dml.setDatabase(commonMessage.getDatabase());
dml.setTable(flatMessage.getTable()); dml.setTable(commonMessage.getTable());
dml.setPkNames(flatMessage.getPkNames()); dml.setPkNames(commonMessage.getPkNames());
dml.setIsDdl(flatMessage.getIsDdl()); dml.setIsDdl(commonMessage.getIsDdl());
dml.setType(flatMessage.getType()); dml.setType(commonMessage.getType());
dml.setTs(flatMessage.getTs()); dml.setTs(commonMessage.getTs());
dml.setEs(flatMessage.getEs()); dml.setEs(commonMessage.getEs());
dml.setSql(flatMessage.getSql()); dml.setSql(commonMessage.getSql());
// if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) { // if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
// throw new RuntimeException("SqlType or mysqlType is null"); // throw new RuntimeException("SqlType or mysqlType is null");
// } // }
List<Map<String, String>> data = flatMessage.getData(); List<Map<String, Object>> data = commonMessage.getData();
if (data != null) { if (data != null) {
dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType())); dml.setData(data);
} }
List<Map<String, String>> old = flatMessage.getOld(); List<Map<String, Object>> old = commonMessage.getOld();
if (old != null) { if (old != null) {
dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType())); dml.setOld(old);
} }
return dml; return dml;
} }

77
client-adapter/launcher/pom.xml
View File

@@ -30,11 +30,6 @@
<artifactId>client-adapter.common</artifactId> <artifactId>client-adapter.common</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.yaml</groupId> <groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId> <artifactId>snakeyaml</artifactId>
@@ -56,25 +51,6 @@
<groupId>org.apache.curator</groupId> <groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId> <artifactId>curator-recipes</artifactId>
</dependency> </dependency>
<!-- 单独引入rocketmq依赖 -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</dependency>
<!-- 单独引入kafka依赖 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
</dependency>
<!-- jdbc --> <!-- jdbc -->
<dependency> <dependency>
<groupId>mysql</groupId> <groupId>mysql</groupId>
@@ -159,6 +135,59 @@
<classifier>jar-with-dependencies</classifier> <classifier>jar-with-dependencies</classifier>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<!-- connector plugin -->
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.tcp</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.kafka</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rocketmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>

263
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -1,263 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.CanalMQConnector;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
* 适配器工作线程抽象类
*
* @author rewerma 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public abstract class AbstractCanalAdapterWorker {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected String canalDestination; // canal实例
protected String groupId = null; // groupId
protected List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
protected CanalClientConfig canalClientConfig; // 配置
protected ExecutorService groupInnerExecutorService; // 组内工作线程池
protected volatile boolean running = false; // 是否运行中
protected Thread thread = null;
protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
protected SyncSwitch syncSwitch;
public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
}
protected void writeOut(final Message message) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
canalOuterAdapters.forEach(outerAdapters -> {
final List<OuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
adapters.forEach(adapter -> {
long begin = System.currentTimeMillis();
List<Dml> dmls = MessageUtil.parse4Dml(canalDestination, groupId, message);
if (dmls != null) {
batchSync(dmls, adapter);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
adapter.getClass().getName(),
(System.currentTimeMillis() - begin));
}
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
RuntimeException exception = null;
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
exception = new RuntimeException("Outer adapter sync failed! ");
}
} catch (Exception e) {
exception = new RuntimeException(e);
}
}
if (exception != null) {
throw exception;
}
});
}
protected void writeOut(final List<FlatMessage> flatMessages) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
canalOuterAdapters.forEach(outerAdapters -> {
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
outerAdapters.forEach(adapter -> {
long begin = System.currentTimeMillis();
List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages);
batchSync(dmls, adapter);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
adapter.getClass().getName(),
(System.currentTimeMillis() - begin));
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
RuntimeException exception = null;
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
exception = new RuntimeException("Outer adapter sync failed! ");
}
} catch (Exception e) {
exception = new RuntimeException(e);
}
}
if (exception != null) {
throw exception;
}
});
}
@SuppressWarnings("unchecked")
protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
CanalMQConnector connector) {
ExecutorService workerExecutor = Util.newSingleDaemonThreadExecutor(5000);
try {
List<?> messages;
if (!flatMessage) {
messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
} else {
messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null && !messages.isEmpty()) {
Future<Boolean> future = workerExecutor.submit(() -> {
if (flatMessage) {
// batch write
writeOut((List<FlatMessage>) messages);
// FIXME xxx
// messages.forEach((message ->
// System.out.println(JSON.toJSONString(message))));
} else {
for (final Object message : messages) {
writeOut((Message) message);
}
}
return true;
});
try {
future.get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
future.cancel(true);
throw e;
}
connector.ack();
}
return true;
} catch (Throwable e) {
if (i == retry - 1) {
connector.ack();
logger.error(e.getMessage() + " Error sync but ACK!");
return true;
} else {
connector.rollback();
logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
}
try {
Thread.sleep(500);
} catch (InterruptedException e1) {
// ignore
}
} finally {
workerExecutor.shutdown();
}
return false;
}
/**
* 分批同步
*
* @param dmls
* @param adapter
*/
private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
// 分批同步
if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmls);
} else {
int len = 0;
List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) {
dmlsBatch.add(dml);
if (dml.getData() == null || dml.getData().isEmpty()) {
len += 1;
} else {
len += dml.getData().size();
}
if (len >= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmlsBatch);
dmlsBatch.clear();
len = 0;
}
}
if (!dmlsBatch.isEmpty()) {
adapter.sync(dmlsBatch);
}
}
}
public void start() {
if (!running) {
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
}
protected abstract void process();
public void stop() {
try {
if (!running) {
return;
}
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}

268
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java Normal file
View File

@@ -0,0 +1,268 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
/**
* 适配处理器
*
* @author rewerma 2020-02-01
* @version 1.0.0
*/
public class AdapterProcessor {
private static final Logger logger = LoggerFactory.getLogger(AdapterProcessor.class);
private static final String CONNECTOR_SPI_DIR = "/plugin";
private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
private CanalMsgConsumer canalMsgConsumer;
private String canalDestination; // canal实例
private String groupId = null; // groupId
private List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
private CanalClientConfig canalClientConfig; // 配置
private ExecutorService groupInnerExecutorService; // 组内工作线程池
private volatile boolean running = false; // 是否运行中
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger
.error("parse events has an error", e);
private SyncSwitch syncSwitch;
public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
List<List<OuterAdapter>> canalOuterAdapters){
this.canalClientConfig = canalClientConfig;
this.canalDestination = destination;
this.groupId = groupId;
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
// load connector consumer
ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
canalMsgConsumer = loader
.getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
Properties properties = canalClientConfig.getConsumerProperties();
properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey());
properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey());
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
canalMsgConsumer.init(properties, canalDestination, groupId);
Thread.currentThread().setContextClassLoader(cl);
}
public void start() {
if (!running) {
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
}
public void writeOut(final List<CommonMessage> commonMessages) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
canalOuterAdapters.forEach(outerAdapters -> {
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
outerAdapters.forEach(adapter -> {
long begin = System.currentTimeMillis();
List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, commonMessages);
batchSync(dmls, adapter);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
adapter.getClass().getName(),
(System.currentTimeMillis() - begin));
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
RuntimeException exception = null;
for (Future<Boolean> future : futures) {
try {
if (!future.get()) {
exception = new RuntimeException("Outer adapter sync failed! ");
}
} catch (Exception e) {
exception = new RuntimeException(e);
}
}
if (exception != null) {
throw exception;
}
});
}
/**
* 分批同步
*
* @param dmls
* @param adapter
*/
private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
// 分批同步
if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmls);
} else {
int len = 0;
List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) {
dmlsBatch.add(dml);
if (dml.getData() == null || dml.getData().isEmpty()) {
len += 1;
} else {
len += dml.getData().size();
}
if (len >= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmlsBatch);
dmlsBatch.clear();
len = 0;
}
}
if (!dmlsBatch.isEmpty()) {
adapter.sync(dmlsBatch);
}
}
}
private void process() {
while (!running) { // waiting until running == true
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
if (retry == -1) {
// 重试次数-1代表异常时一直阻塞重试
retry = Integer.MAX_VALUE;
}
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
canalMsgConsumer.connect();
logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
while (running) {
try {
syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
} catch (TimeoutException e) {
break;
}
if (!running) {
break;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
try {
if (logger.isDebugEnabled()) {
logger.debug("destination: {} ", canalDestination);
}
long begin = System.currentTimeMillis();
List<CommonMessage> commonMessages = canalMsgConsumer
.getMessage(this.canalClientConfig.getTimeout(), TimeUnit.MILLISECONDS);
writeOut(commonMessages);
canalMsgConsumer.ack();
if (logger.isDebugEnabled()) {
logger.debug("destination: {} elapsed time: {} ms",
canalDestination,
System.currentTimeMillis() - begin);
}
} catch (Exception e) {
if (i != retry - 1) {
canalMsgConsumer.rollback(); // 处理失败, 回滚数据
logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
} else {
canalMsgConsumer.ack();
logger.error(e.getMessage() + " Error sync but ACK!");
}
Thread.sleep(500);
}
}
}
canalMsgConsumer.disconnect();
} catch (Throwable e) {
logger.error("process error!", e);
}
if (running) { // is reconnect
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
}
}
}
public void stop() {
try {
if (!running) {
return;
}
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}

96
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -1,96 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
/**
* kafka对应的client适配器工作线程
*
* @author rewerma 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private KafkaCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
super.canalDestination = topic;
super.groupId = groupId;
this.flatMessage = flatMessage;
this.connector = new KafkaCanalConnector(bootstrapServers,
topic,
null,
groupId,
canalClientConfig.getBatchSize(),
flatMessage);
connector.setSessionTimeout(30L, TimeUnit.SECONDS);
}
@Override
protected void process() {
while (!running) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
// ignore
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
while (running) {
boolean status = syncSwitch.status(canalDestination);
if (!status) {
connector.disconnect();
break;
}
if (retry == -1) {
retry = Integer.MAX_VALUE;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
break;
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

255
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -37,9 +37,7 @@ public class CanalAdapterLoader {
private CanalClientConfig canalClientConfig; private CanalClientConfig canalClientConfig;
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>(); private Map<String, AdapterProcessor> canalAdapterProcessors = new HashMap<>();
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private ExtensionLoader<OuterAdapter> loader; private ExtensionLoader<OuterAdapter> loader;
@@ -53,120 +51,132 @@ public class CanalAdapterLoader {
public void init() { public void init() {
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
String canalServerHost = this.canalClientConfig.getCanalServerHost(); for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
SocketAddress sa = null; for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
if (canalServerHost != null) {
String[] ipPort = canalServerHost.split(":");
sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
}
String zkHosts = this.canalClientConfig.getZookeeperHosts();
if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { for (OuterAdapterConfig config : group.getOuterAdapters()) {
List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>(); loadAdapter(config, canalOuterAdapters);
for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
loadAdapter(c, canalOutConnectors);
}
canalOuterAdapterGroups.add(canalOutConnectors);
} }
CanalAdapterWorker worker; canalOuterAdapterGroups.add(canalOuterAdapters);
if (sa != null) {
worker = new CanalAdapterWorker(canalClientConfig,
canalAdapter.getInstance(),
sa,
canalOuterAdapterGroups);
} else if (zkHosts != null) {
worker = new CanalAdapterWorker(canalClientConfig,
canalAdapter.getInstance(),
zkHosts,
canalOuterAdapterGroups);
} else {
throw new RuntimeException("No canal server connector found");
}
canalWorkers.put(canalAdapter.getInstance(), worker);
worker.start();
logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
}
} else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-kafka的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()+"|"+StringUtils.trimToEmpty(group.getGroupId()),
canalClientConfig.getMqServers(), f-> new AdapterProcessor(canalClientConfig,canalAdapter.getInstance(),
canalAdapter.getInstance(), group.getGroupId(), canalOuterAdapterGroups));
group.getGroupId(), adapterProcessor.start();
canalOuterAdapterGroups,
canalClientConfig.getFlatMessage()); logger.info("Start adapter for canal-client mq topic: {} succeed",
canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId()); canalAdapter.getInstance() + "-" + group.getGroupId());
}
} }
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-rocketMQ的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getMqServers(),
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getAccessKey(),
canalClientConfig.getSecretKey(),
canalClientConfig.getFlatMessage(),
canalClientConfig.isEnableMessageTrace(),
canalClientConfig.getCustomizedTraceTopic(),
canalClientConfig.getAccessChannel(),
canalClientConfig.getNamespace());
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId());
}
}
} else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-rabbitMQ的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
canalOuterAdapterGroups,
canalAdapter.getInstance(),
group.getGroupId(),
canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
rabbitMQWork.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId());
}
}
// CanalAdapterRabbitMQWork
} }
// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
//
// for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
// List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
// loadAdapter(c, canalOutConnectors);
// }
// canalOuterAdapterGroups.add(canalOutConnectors);
// }
// CanalAdapterWorker worker;
// if (StringUtils.isNotEmpty(canalServerHost)) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// canalServerHost,
// zkHosts,
// canalOuterAdapterGroups);
// } else if (zkHosts != null) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// zkHosts,
// canalOuterAdapterGroups);
// } else {
// throw new RuntimeException("No canal server connector found");
// }
// canalWorkers.put(canalAdapter.getInstance(), worker);
// worker.start();
// logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
// }
// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-kafka的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
//
// CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
// canalKafkaWorker.start();
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-rocketMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getAccessKey(),
// canalClientConfig.getSecretKey(),
// canalClientConfig.getFlatMessage(),
// canalClientConfig.isEnableMessageTrace(),
// canalClientConfig.getCustomizedTraceTopic(),
// canalClientConfig.getAccessChannel(),
// canalClientConfig.getNamespace());
// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
// rocketMQWorker.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-rabbitMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
// canalOuterAdapterGroups,
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
// rabbitMQWork.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// // CanalAdapterRabbitMQWork
// }
} }
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) { private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
@@ -206,10 +216,10 @@ public class CanalAdapterLoader {
* 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
*/ */
public void destroy() { public void destroy() {
if (!canalWorkers.isEmpty()) { if (!canalAdapterProcessors.isEmpty()) {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalAdapterProcessors.size());
for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { for (AdapterProcessor adapterProcessor : canalAdapterProcessors.values()) {
stopExecutorService.execute(canalAdapterWorker::stop); stopExecutorService.execute(adapterProcessor::stop);
} }
stopExecutorService.shutdown(); stopExecutorService.shutdown();
try { try {
@@ -220,21 +230,6 @@ public class CanalAdapterLoader {
// ignore // ignore
} }
} }
if (!canalMQWorker.isEmpty()) {
ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
stopMQWorkerService.execute(canalAdapterMQWorker::stop);
}
stopMQWorkerService.shutdown();
try {
while (!stopMQWorkerService.awaitTermination(1, TimeUnit.SECONDS)) {
// ignore
}
} catch (InterruptedException e) {
// ignore
}
}
logger.info("All canal adapters destroyed"); logger.info("All canal adapters destroyed");
} }
} }

90
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java
View File

@@ -1,90 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector;
import org.apache.kafka.common.errors.WakeupException;
import java.util.List;
import java.util.concurrent.ExecutorService;
public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker {
private RabbitMQCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List<List<OuterAdapter>> canalOuterAdapters,
String topic, String groupId, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
this.canalDestination = topic;
this.groupId = groupId;
this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(),
canalClientConfig.getVhost(),
topic,
canalClientConfig.getAccessKey(),
canalClientConfig.getSecretKey(),
canalClientConfig.getUsername(),
canalClientConfig.getPassword(),
canalClientConfig.getResourceOwnerId(),
flatMessage);
}
@Override
protected void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
boolean status = syncSwitch.status(canalDestination);
if (!status) {
connector.disconnect();
break;
}
if (retry == -1) {
retry = Integer.MAX_VALUE;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
break;
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

119
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -1,119 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
/**
* rocketmq对应的client适配器工作线程
*
* @version 1.0.0
*/
public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private RocketMQCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
String secretKey, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
super.canalDestination = topic;
super.groupId = groupId;
this.connector = new RocketMQCanalConnector(nameServers,
topic,
groupId,
accessKey,
secretKey,
canalClientConfig.getBatchSize(),
flatMessage);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
}
public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
String secretKey, boolean flatMessage, boolean enableMessageTrace,
String customizedTraceTopic, String accessChannel, String namespace) {
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
super.canalDestination = topic;
super.groupId = groupId;
this.connector = new RocketMQCanalConnector(nameServers,
topic,
groupId,
accessKey,
secretKey,
canalClientConfig.getBatchSize(),
flatMessage,
enableMessageTrace,
customizedTraceTopic,
accessChannel,
namespace);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
}
@Override
protected void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
boolean status = syncSwitch.status(canalDestination);
if (!status) {
connector.disconnect();
break;
}
if (retry == -1) {
retry = Integer.MAX_VALUE;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
break;
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

193
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -1,193 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.SocketAddress;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
/**
* 原生canal-server对应的client适配器工作线程
*
* @author rewrema 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
private static final int BATCH_SIZE = 50;
private static final int SO_TIMEOUT = 0;
private CanalConnector connector;
/**
* 单台client适配器worker的构造方法
*
* @param canalDestination canal实例名
* @param address canal-server地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.canalDestination = canalDestination;
connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
}
/**
* HA模式下client适配器worker的构造方法
*
* @param canalDestination canal实例名
* @param zookeeperHosts zookeeper地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalDestination = canalDestination;
this.canalClientConfig = canalClientConfig;
connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
}
@Override
protected void process() {
while (!running) { // waiting until running == true
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
}
int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
if (retry == -1) {
// 重试次数-1代表异常时一直阻塞重试
retry = Integer.MAX_VALUE;
}
// long timeout = canalClientConfig.getTimeout() == null ? 300000 :
// canalClientConfig.getTimeout(); // 默认超时5分钟
Integer batchSize = canalClientConfig.getBatchSize();
if (batchSize == null) {
batchSize = BATCH_SIZE;
}
while (running) {
try {
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
connector.connect();
logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination);
connector.subscribe();
logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
while (running) {
try {
syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
} catch (TimeoutException e) {
break;
}
if (!running) {
break;
}
for (int i = 0; i < retry; i++) {
if (!running) {
break;
}
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
try {
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
Thread.sleep(500);
} else {
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} batchSize: {} ",
canalDestination,
batchId,
size);
}
long begin = System.currentTimeMillis();
writeOut(message);
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} elapsed time: {} ms",
canalDestination,
batchId,
System.currentTimeMillis() - begin);
}
}
connector.ack(batchId); // 提交确认
break;
} catch (Exception e) {
if (i != retry - 1) {
connector.rollback(batchId); // 处理失败, 回滚数据
logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
} else {
connector.ack(batchId);
logger.error(e.getMessage() + " Error sync but ACK!");
}
Thread.sleep(500);
}
}
}
} catch (Throwable e) {
logger.error("process error!", e);
} finally {
connector.disconnect();
logger.info("=============> Disconnect destination: {} <=============", this.canalDestination);
}
if (running) { // is reconnect
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
}
}
}
@Override
public void stop() {
try {
if (!running) {
return;
}
if (connector instanceof ClusterCanalConnector) {
((ClusterCanalConnector) connector).stopRunning();
} else if (connector instanceof SimpleCanalConnector) {
((SimpleCanalConnector) connector).stopRunning();
}
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}

4
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -15,14 +15,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder; import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
/** /**

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java
View File

@@ -5,11 +5,10 @@ import java.io.FileOutputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.utils.CommonUtils;
/** /**
* 远程配置监听器实现 * 远程配置监听器实现
* *

43
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -7,20 +7,45 @@ spring:
default-property-inclusion: non_null default-property-inclusion: non_null
canal.conf: canal.conf:
mode: tcp # kafka rocketMQ mode: tcp #tcp kafka rocketMQ rabbitMQ
canalServerHost: 127.0.0.1:11111 flatMessage: true
# zookeeperHosts: slave1:2181 zookeeperHosts:
# mqServers: 127.0.0.1:9092 #or rocketmq
# flatMessage: true
batchSize: 500
syncBatchSize: 1000 syncBatchSize: 1000
retries: 0 retries: 0
timeout: timeout:
accessKey: accessKey:
secretKey: secretKey:
username: consumerProperties:
password: # canal tcp consumer
vhost: canal.tcp.server.host: 127.0.0.1:11111
canal.tcp.zookeeper.hosts:
canal.tcp.batch.size: 500
canal.tcp.username:
canal.tcp.password:
# kafka consumer
kafka.bootstrap.servers: 127.0.0.1:9092
kafka.enable.auto.commit: false
kafka.auto.commit.interval.ms: 1000
kafka.auto.offset.reset: latest
kafka.request.timeout.ms: 40000
kafka.session.timeout.ms: 30000
kafka.isolation.level: read_committed
kafka.max.poll.records: 1000
# rocketMQ consumer
rocketmq.namespace:
rocketmq.namesrv.addr: 127.0.0.1:9876
rocketmq.batch.size: 1000
rocketmq.enable.message.trace: false
rocketmq.customized.trace.topic:
rocketmq.access.channel:
rocketmq.subscribe.filter:
# rabbitMQ consumer
rabbitmq.host:
rabbitmq.virtual.host:
rabbitmq.username:
rabbitmq.password:
rabbitmq.resource.ownerId:
# srcDataSources: # srcDataSources:
# defaultDS: # defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true # url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true

12
client/pom.xml
View File

@@ -101,23 +101,31 @@
<version>${spring_version}</version> <version>${spring_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- 客户端要使用请单独引入mq-clients依赖 -->
<dependency> <dependency>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId> <artifactId>rocketmq-client</artifactId>
<version>4.5.2</version>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId> <artifactId>rocketmq-acl</artifactId>
<version>4.5.2</version>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.rabbitmq</groupId> <groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId> <artifactId>amqp-client</artifactId>
<version>5.5.0</version>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.alibaba.mq-amqp</groupId> <groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId> <artifactId>mq-amqp-client</artifactId>
<version>1.0.3</version>
<scope>provided</scope>
</dependency> </dependency>
<!-- 客户端要使用请单独引入kafka-clients依赖 -->
<dependency> <dependency>
<groupId>org.apache.kafka</groupId> <groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId> <artifactId>kafka-clients</artifactId>
@@ -143,7 +151,7 @@
<createDependencyReducedPom>false</createDependencyReducedPom> <createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet> <artifactSet>
<includes> <includes>
<include>com.alibaba.otter:*</include> <include>com.alibaba.otter:canal.client</include>
<include>com.google.guava:*</include> <include>com.google.guava:*</include>
<include>org.joda:*</include> <include>org.joda:*</include>
<include>com.google.common:*</include> <include>com.google.common:*</include>

40
connector/core/pom.xml Normal file
View File

@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<packaging>jar</packaging>
<name>canal connector core module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>oro</groupId>
<artifactId>oro</artifactId>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.4</version>
</dependency>
</dependencies>
</project>

26
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java Normal file
View File

@@ -0,0 +1,26 @@
package com.alibaba.otter.canal.connector.core.config;
/**
* MQ配置常量
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class CanalConstants {
public static final String ROOT = "canal";
public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
public static final String CANAL_MQ_FLAT_MESSAGE = ROOT + "." + "mq.flat.message";
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
public static final String CANAL_MQ_PARALLEL_THREAD_SIZE = ROOT + "." + "mq.parallel.thread.size";
public static final String CANAL_MQ_CANAL_BATCH_SIZE = ROOT + "." + "mq.canal.batch.size";
public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT = ROOT + "." + "mq.canal.fetch.timeout";
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.access.channel";
public static final String CANAL_ALIYUN_ACCESS_KEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRET_KEY = ROOT + "." + "aliyun.secretKey";
public static final String CANAL_ALIYUN_UID = ROOT + "." + "aliyun.uid";
}

102
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java Normal file
View File

@@ -0,0 +1,102 @@
package com.alibaba.otter.canal.connector.core.config;
/**
* MQ配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class MQProperties {
private boolean flatMessage = true;
private boolean databaseHash = true;
private boolean filterTransactionEntry = true;
private Integer parallelThreadSize = 8;
private Integer fetchTimeout = 100;
private Integer batchSize = 50;
private String accessChannel = "local";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private int aliyunUid = 0;
public boolean isFlatMessage() {
return flatMessage;
}
public void setFlatMessage(boolean flatMessage) {
this.flatMessage = flatMessage;
}
public boolean isDatabaseHash() {
return databaseHash;
}
public void setDatabaseHash(boolean databaseHash) {
this.databaseHash = databaseHash;
}
public boolean isFilterTransactionEntry() {
return filterTransactionEntry;
}
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
this.filterTransactionEntry = filterTransactionEntry;
}
public Integer getParallelThreadSize() {
return parallelThreadSize;
}
public void setParallelThreadSize(Integer parallelThreadSize) {
this.parallelThreadSize = parallelThreadSize;
}
public Integer getFetchTimeout() {
return fetchTimeout;
}
public void setFetchTimeout(Integer fetchTimeout) {
this.fetchTimeout = fetchTimeout;
}
public Integer getBatchSize() {
return batchSize;
}
public void setBatchSize(Integer batchSize) {
this.batchSize = batchSize;
}
public String getAccessChannel() {
return accessChannel;
}
public void setAccessChannel(String accessChannel) {
this.accessChannel = accessChannel;
}
public String getAliyunAccessKey() {
return aliyunAccessKey;
}
public void setAliyunAccessKey(String aliyunAccessKey) {
this.aliyunAccessKey = aliyunAccessKey;
}
public String getAliyunSecretKey() {
return aliyunSecretKey;
}
public void setAliyunSecretKey(String aliyunSecretKey) {
this.aliyunSecretKey = aliyunSecretKey;
}
public int getAliyunUid() {
return aliyunUid;
}
public void setAliyunUid(int aliyunUid) {
this.aliyunUid = aliyunUid;
}
}

131
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java Normal file
View File

@@ -0,0 +1,131 @@
package com.alibaba.otter.canal.connector.core.consumer;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
public class CommonMessage implements Serializable {
private static final long serialVersionUID = 2611556444074013268L;
private String database; // 数据库或schema
private String table; // 表名
private List<String> pkNames;
private Boolean isDdl;
private String type; // 类型: INSERT UPDATE DELETE
// binlog executeTime
private Long es; // 执行耗时
// dml build timeStamp
private Long ts; // 同步时间
private String sql; // 执行的sql, dml sql为空
private List<Map<String, Object>> data; // 数据列表
private List<Map<String, Object>> old; // 旧数据列表, 用于update, size和data的size一一对应
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public List<String> getPkNames() {
return pkNames;
}
public void setPkNames(List<String> pkNames) {
this.pkNames = pkNames;
}
public Boolean getIsDdl() {
return isDdl;
}
public void setIsDdl(Boolean isDdl) {
this.isDdl = isDdl;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Long getTs() {
return ts;
}
public void setTs(Long ts) {
this.ts = ts;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<Map<String, Object>> getData() {
return data;
}
public void setData(List<Map<String, Object>> data) {
this.data = data;
}
public List<Map<String, Object>> getOld() {
return old;
}
public void setOld(List<Map<String, Object>> old) {
this.old = old;
}
public Long getEs() {
return es;
}
public void setEs(Long es) {
this.es = es;
}
public void clear() {
database = null;
table = null;
type = null;
ts = null;
es = null;
data = null;
old = null;
sql = null;
}
@Override
public String toString() {
return "CommonMessage{" +
"database='" + database + '\'' +
", table='" + table + '\'' +
", pkNames=" + pkNames +
", isDdl=" + isDdl +
", type='" + type + '\'' +
", es=" + es +
", ts=" + ts +
", sql='" + sql + '\'' +
", data=" + data +
", old=" + old +
'}';
}
}

125
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java Normal file
View File

@@ -0,0 +1,125 @@
package com.alibaba.otter.canal.connector.core.filter;
import java.util.*;
import org.apache.commons.lang.StringUtils;
import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.Expression;
/**
* 基于aviater进行tableName正则匹配的过滤算法
*
* @author jianghang 2012-7-20 下午06:01:34
*/
public class AviaterRegexFilter {
private static final String SPLIT = ",";
private static final String PATTERN_SPLIT = "|";
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
private static final RegexFunction regexFunction = new RegexFunction();
private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
static {
AviatorEvaluator.addFunction(regexFunction);
}
private static final Comparator<String> COMPARATOR = new StringComparator();
final private String pattern;
final private boolean defaultEmptyValue;
public AviaterRegexFilter(String pattern){
this(pattern, true);
}
public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null;
if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>();
} else {
String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss);
}
// 对pattern按照从长到短的排序
// 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
// 的长度不一样
Collections.sort(list, COMPARATOR);
// 对pattern进行头尾完全匹配
list = completionPattern(list);
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
}
public boolean filter(String filtered) {
if (StringUtils.isEmpty(pattern)) {
return defaultEmptyValue;
}
if (StringUtils.isEmpty(filtered)) {
return defaultEmptyValue;
}
Map<String, Object> env = new HashMap<String, Object>();
env.put("pattern", pattern);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
}
/**
* 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
*
* <pre>
* foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
* </pre>
*
* 因此此类对正则表达式进行了从长到短的排序
*
* @author zebin.xuzb 2012-10-22 下午2:02:26
* @version 1.0.0
*/
private static class StringComparator implements Comparator<String> {
@Override
public int compare(String str1, String str2) {
if (str1.length() > str2.length()) {
return -1;
} else if (str1.length() < str2.length()) {
return 1;
} else {
return 0;
}
}
}
/**
* 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
*
* <pre>
* foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
* </pre>
*
* 因此此类对正则表达式进行头尾完全匹配
*
* @author simon
* @version 1.0.0
*/
private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>();
for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^");
stringBuffer.append(pattern);
stringBuffer.append("$");
result.add(stringBuffer.toString());
}
return result;
}
@Override
public String toString() {
return pattern;
}
}

33
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java Normal file
View File

@@ -0,0 +1,33 @@
package com.alibaba.otter.canal.connector.core.filter;
import java.util.Map;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
public class PatternUtils {
@SuppressWarnings("deprecation")
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e) {
throw new RuntimeException(e);
}
});
public static Pattern getPattern(String pattern) {
return patterns.get(pattern);
}
public static void clear() {
patterns.clear();
}
}

31
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java Normal file
View File

@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.connector.core.filter;
import java.util.Map;
import org.apache.oro.text.regex.Perl5Matcher;
import com.googlecode.aviator.runtime.function.AbstractFunction;
import com.googlecode.aviator.runtime.function.FunctionUtils;
import com.googlecode.aviator.runtime.type.AviatorBoolean;
import com.googlecode.aviator.runtime.type.AviatorObject;
/**
* 提供aviator regex的代码扩展
*
* @author jianghang 2012-7-23 上午10:29:23
*/
public class RegexFunction extends AbstractFunction {
public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
String pattern = FunctionUtils.getStringValue(arg1, env);
String text = FunctionUtils.getStringValue(arg2, env);
Perl5Matcher matcher = new Perl5Matcher();
boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
return AviatorBoolean.valueOf(isMatch);
}
public String getName() {
return "regex";
}
}

110
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java Normal file
View File

@@ -0,0 +1,110 @@
package com.alibaba.otter.canal.connector.core.producer;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
/**
* MQ producer 抽象类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public abstract class AbstractMQProducer implements CanalMQProducer {
protected MQProperties mqProperties;
protected ThreadPoolExecutor executor;
@Override
public void init(Properties properties) {
// parse canal mq properties
loadCanalMqProperties(properties);
int parallelThreadSize = mqProperties.getParallelThreadSize();
executor = new ThreadPoolExecutor(parallelThreadSize,
parallelThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
new NamedThreadFactory("MQParallel"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@Override
public MQProperties getMqProperties() {
return this.mqProperties;
}
@Override
public void stop() {
if (executor != null) {
executor.shutdownNow();
}
}
/**
* 初始化配置
* <p>
* canal.mq.flat.message = true <br/>
* canal.mq.database.hash = true <br/>
* canal.mq.filter.transaction.entry = true <br/>
* canal.mq.parallel.thread.size = 8 <br/>
* canal.mq.batch.size = 50 <br/>
* canal.mq.timeout = 100 <br/>
* canal.mq.access.channel = local <br/>
* </p>
*
* @param properties 总配置对象
*/
private void loadCanalMqProperties(Properties properties) {
String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
}
String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
if (!StringUtils.isEmpty(databaseHash)) {
mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
}
String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
if (!StringUtils.isEmpty(filterTranEntry)) {
mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
}
String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelThreadSize)) {
mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize));
}
String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
if (!StringUtils.isEmpty(batchSize)) {
mqProperties.setBatchSize(Integer.parseInt(batchSize));
}
String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT);
if (!StringUtils.isEmpty(timeOut)) {
mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
}
String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
if (!StringUtils.isEmpty(accessChannel)) {
mqProperties.setAccessChannel(accessChannel);
}
String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunAccessKey(aliyunSecretKey);
}
String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
if (!StringUtils.isEmpty(aliyunUid)) {
mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
}
}
}

64
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java Normal file
View File

@@ -0,0 +1,64 @@
package com.alibaba.otter.canal.connector.core.producer;
/**
* MQ producer destination
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class MQDestination {
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private String partitionHash;
private String dynamicTopic;
public String getCanalDestination() {
return canalDestination;
}
public void setCanalDestination(String canalDestination) {
this.canalDestination = canalDestination;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Integer getPartition() {
return partition;
}
public void setPartition(Integer partition) {
this.partition = partition;
}
public Integer getPartitionsNum() {
return partitionsNum;
}
public void setPartitionsNum(Integer partitionsNum) {
this.partitionsNum = partitionsNum;
}
public String getPartitionHash() {
return partitionHash;
}
public void setPartitionHash(String partitionHash) {
this.partitionHash = partitionHash;
}
public String getDynamicTopic() {
return dynamicTopic;
}
public void setDynamicTopic(String dynamicTopic) {
this.dynamicTopic = dynamicTopic;
}
}

136
server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java → connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
View File

@@ -1,25 +1,18 @@
package com.alibaba.otter.canal.common; package com.alibaba.otter.canal.connector.core.producer;
import java.util.ArrayList; import java.util.*;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import com.google.common.base.Function;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate; import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry; import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.google.common.base.Function;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker; import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
@@ -29,77 +22,88 @@ import com.google.protobuf.InvalidProtocolBufferException;
/** /**
* process MQ Message utils * process MQ Message utils
* *
* @author agapple 2018年12月11日 下午1:28:32
*/
/**
* @author agapple 2019年9月29日 下午12:36:26 * @author agapple 2019年9月29日 下午12:36:26
* @since 5.0.0 * @since 5.0.0
*/ */
public class MQMessageUtils { public class MQMessageUtils {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(), private static Map<String, List<PartitionData>> partitionDatas = MigrateMap
new Function<String, List<PartitionData>>() { .makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> {
List<PartitionData> datas = Lists
.newArrayList();
public List<PartitionData> apply(String pkHashConfigs) { String[] pkHashConfigArray = StringUtils
List<PartitionData> datas = Lists.newArrayList(); .split(StringUtils.replace(
pkHashConfigs,
",",
";"), ";");
// schema.table:id^name
for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData();
int i = pkHashConfig
.lastIndexOf(":");
if (i > 0) {
String pkStr = pkHashConfig
.substring(i + 1);
if (pkStr.equalsIgnoreCase(
"$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists
.newArrayList(
StringUtils
.split(
pkStr,
'^'));
}
String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs, pkHashConfig = pkHashConfig
",", .substring(0, i);
";"), } else {
";"); data.hashMode.tableHash = true;
// schema.table:id^name }
for (String pkHashConfig : pkHashConfigArray) {
PartitionData data = new PartitionData();
int i = pkHashConfig.lastIndexOf(":");
if (i > 0) {
String pkStr = pkHashConfig.substring(i + 1);
if (pkStr.equalsIgnoreCase("$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
'^'));
}
pkHashConfig = pkHashConfig.substring(0, if (!isWildCard(
i); pkHashConfig)) {
} else { data.simpleName = pkHashConfig;
data.hashMode.tableHash = true; } else {
} data.regexFilter = new AviaterRegexFilter(
pkHashConfig);
}
datas.add(data);
}
if (!isWildCard(pkHashConfig)) { return datas;
data.simpleName = pkHashConfig; });
} else {
data.regexFilter = new AviaterRegexFilter(pkHashConfig);
}
datas.add(data);
}
return datas;
}
});
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(), private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap
new Function<String, List<DynamicTopicData>>() { .makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) { public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists.newArrayList(); List<DynamicTopicData> datas = Lists
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs, .newArrayList();
",", String[] dynamicTopicArray = StringUtils
";"), .split(StringUtils.replace(
";"); pkHashConfigs,
",",
";"), ";");
// schema.table // schema.table
for (String dynamicTopic : dynamicTopicArray) { for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData(); DynamicTopicData data = new DynamicTopicData();
if (!isWildCard(dynamicTopic)) { if (!isWildCard(
dynamicTopic)) {
data.simpleName = dynamicTopic; data.simpleName = dynamicTopic;
} else { } else {
if (dynamicTopic.contains("\\.")) { if (dynamicTopic
data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic); .contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} else { } else {
data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic); data.schemaRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} }
} }
datas.add(data); datas.add(data);
@@ -238,8 +242,8 @@ public class MQMessageUtils {
* @return 分区message数组 * @return 分区message数组
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs,
String pkHashConfigs, boolean databaseHash) { boolean databaseHash) {
if (partitionsNum == null) { if (partitionsNum == null) {
partitionsNum = 1; partitionsNum = 1;
} }
@@ -338,8 +342,8 @@ public class MQMessageUtils {
/** /**
* 将Message转换为FlatMessage * 将Message转换为FlatMessage
* *
* @param message 原生message
* @return FlatMessage列表 * @return FlatMessage列表
* @author agapple 2018年12月11日 下午1:28:32
*/ */
public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) { public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
List<FlatMessage> flatMessages = new ArrayList<>(); List<FlatMessage> flatMessages = new ArrayList<>();
@@ -627,8 +631,8 @@ public class MQMessageUtils {
private static boolean isWildCard(String value) { private static boolean isWildCard(String value) {
// not contaiins '.' ? // not contaiins '.' ?
return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', return StringUtils.containsAny(value,
'^' }); new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' });
} }
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName, private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,

43
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java Normal file
View File

@@ -0,0 +1,43 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.util.Properties;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.protocol.Message;
/**
* MQ producer SPI 接口
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
@SPI("kafka")
public interface CanalMQProducer {
/**
* Init producer.
*/
void init(Properties properties);
/**
* Get base mq properties
*
* @return MQProperties
*/
MQProperties getMqProperties();
/**
* Send canal message to related topic
*
* @param canalDestination canal mq destination
* @param message canal message
*/
void send(MQDestination canalDestination, Message message, Callback callback);
/**
* Stop MQ producer service
*/
void stop();
}

55
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java Normal file
View File

@@ -0,0 +1,55 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
/**
* Canal/MQ consumer SPI 接口
*
* @author rewerma @ 2020-02-01
* @version 1.0.0
*/
@SPI("kafka")
public interface CanalMsgConsumer {
/**
* 初始化
*
* @param properties consumer properties
* @param topic topic/destination
* @param groupId mq group id
*/
void init(Properties properties, String topic, String groupId);
/**
* 连接Canal/MQ
*/
void connect();
/**
* 批量拉取数据
*
* @param timeout 超时时间
* @param unit 时间单位
* @return Message列表
*/
List<CommonMessage> getMessage(Long timeout, TimeUnit unit);
/**
* 提交
*/
void ack();
/**
* 回滚
*/
void rollback();
/**
* 断开连接
*/
void disconnect();
}

435
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java Normal file
View File

@@ -0,0 +1,435 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SPI 类加载器
*
* @author rewerma 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class ExtensionLoader<T> {
private static final Logger logger = LoggerFactory
.getLogger(ExtensionLoader.class);
private static final String SERVICES_DIRECTORY = "META-INF/services/";
private static final String CANAL_DIRECTORY = "META-INF/canal/";
private static final String DEFAULT_CLASSLOADER_POLICY = "internal";
private static final Pattern NAME_SEPARATOR = Pattern
.compile("\\s*[,]+\\s*");
private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS = new ConcurrentHashMap<>();
private static final ConcurrentMap<Class<?>, Object> EXTENSION_INSTANCES = new ConcurrentHashMap<>();
private static final ConcurrentMap<String, Object> EXTENSION_KEY_INSTANCE = new ConcurrentHashMap<>();
private final Class<?> type;
private final String classLoaderPolicy;
private final ConcurrentMap<Class<?>, String> cachedNames = new ConcurrentHashMap<>();
private final Holder<Map<String, Class<?>>> cachedClasses = new Holder<>();
private final ConcurrentMap<String, Holder<Object>> cachedInstances = new ConcurrentHashMap<>();
private String cachedDefaultName;
private ConcurrentHashMap<String, IllegalStateException> exceptions = new ConcurrentHashMap<>();
private static <T> boolean withExtensionAnnotation(Class<T> type) {
return type.isAnnotationPresent(SPI.class);
}
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
}
@SuppressWarnings("unchecked")
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
if (type == null) throw new IllegalArgumentException("Extension type == null");
if (!type.isInterface()) {
throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
}
if (!withExtensionAnnotation(type)) {
throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+ SPI.class.getSimpleName() + " Annotation!");
}
ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
if (loader == null) {
EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
}
return loader;
}
public ExtensionLoader(Class<?> type){
this.type = type;
this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY;
}
public ExtensionLoader(Class<?> type, String classLoaderPolicy){
this.type = type;
this.classLoaderPolicy = classLoaderPolicy;
}
/**
* 返回指定名字的扩展
*
* @param name
* @return
*/
@SuppressWarnings("unchecked")
public T getExtension(String name, String spiDir, String standbyDir) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
if ("true".equals(name)) {
return getDefaultExtension(spiDir, standbyDir);
}
Holder<Object> holder = cachedInstances.get(name);
if (holder == null) {
cachedInstances.putIfAbsent(name, new Holder<>());
holder = cachedInstances.get(name);
}
Object instance = holder.get();
if (instance == null) {
synchronized (holder) {
instance = holder.get();
if (instance == null) {
instance = createExtension(name, spiDir, standbyDir);
holder.set(instance);
}
}
}
return (T) instance;
}
@SuppressWarnings("unchecked")
public T getExtension(String name, String key, String spiDir, String standbyDir) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
if ("true".equals(name)) {
return getDefaultExtension(spiDir, standbyDir);
}
String extKey = name + "-" + StringUtils.trimToEmpty(key);
Holder<Object> holder = cachedInstances.get(extKey);
if (holder == null) {
cachedInstances.putIfAbsent(extKey, new Holder<>());
holder = cachedInstances.get(extKey);
}
Object instance = holder.get();
if (instance == null) {
synchronized (holder) {
instance = holder.get();
if (instance == null) {
instance = createExtension(name, key, spiDir, standbyDir);
holder.set(instance);
}
}
}
return (T) instance;
}
/**
* 返回缺省的扩展，如果没有设置则返回<code>null</code>
*/
public T getDefaultExtension(String spiDir, String standbyDir) {
getExtensionClasses(spiDir, standbyDir);
if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
return null;
}
return getExtension(cachedDefaultName, spiDir, standbyDir);
}
@SuppressWarnings("unchecked")
public T createExtension(String name, String spiDir, String standbyDir) {
Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: class could not be found");
}
try {
T instance = (T) EXTENSION_INSTANCES.get(clazz);
if (instance == null) {
EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
instance = (T) EXTENSION_INSTANCES.get(clazz);
}
return instance;
} catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(),
t);
}
}
@SuppressWarnings("unchecked")
private T createExtension(String name, String key, String spiDir, String standbyDir) {
Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: class could not be found");
}
try {
T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
if (instance == null) {
EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance());
instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
}
return instance;
} catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(),
t);
}
}
private Map<String, Class<?>> getExtensionClasses(String spiDir, String standbyDir) {
Map<String, Class<?>> classes = cachedClasses.get();
if (classes == null) {
synchronized (cachedClasses) {
classes = cachedClasses.get();
if (classes == null) {
classes = loadExtensionClasses(spiDir, standbyDir);
cachedClasses.set(classes);
}
}
}
return classes;
}
private String getJarDirectoryPath() {
URL url = Thread.currentThread().getContextClassLoader().getResource("");
String dirtyPath;
if (url != null) {
dirtyPath = url.toString();
} else {
File file = new File("");
dirtyPath = file.getAbsolutePath();
}
String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
// file:/ and
// everything
// before it
jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything
// after .jar, if .jar
// exists in dirtyPath
jarPath = jarPath.replaceAll("%20", " "); // necessary if path has
// spaces within
if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run
// the app using Spring Tools Suit play
// button.
jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
}
Path path = Paths.get(jarPath).getParent(); // Paths - from java 8
if (path != null) {
return path.toString();
}
return null;
}
private Map<String, Class<?>> loadExtensionClasses(String spiDir, String standbyDir) {
final SPI defaultAnnotation = type.getAnnotation(SPI.class);
if (defaultAnnotation != null) {
String value = defaultAnnotation.value();
if ((value = value.trim()).length() > 0) {
String[] names = NAME_SEPARATOR.split(value);
if (names.length > 1) {
throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+ ": " + Arrays.toString(names));
}
if (names.length == 1) cachedDefaultName = names[0];
}
}
Map<String, Class<?>> extensionClasses = new HashMap<>();
if (spiDir != null && standbyDir != null) {
// 1. plugin folder，customized extension classLoader （jar_dir/plugin）
String dir = File.separator + this.getJarDirectoryPath() + spiDir; // + "plugin";
File externalLibDir = new File(dir);
if (!externalLibDir.exists()) {
externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir);
}
logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath());
if (externalLibDir.exists()) {
File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar"));
if (files != null) {
for (File f : files) {
URL url;
try {
url = f.toURI().toURL();
} catch (MalformedURLException e) {
throw new RuntimeException("load extension jar failed!", e);
}
ClassLoader parent = Thread.currentThread().getContextClassLoader();
URLClassLoader localClassLoader;
if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
|| DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
localClassLoader = new URLClassExtensionLoader(new URL[] { url });
} else {
localClassLoader = new URLClassLoader(new URL[] { url }, parent);
}
loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
}
}
}
}
// 2. load inner extension class with default classLoader
ClassLoader classLoader = findClassLoader();
loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
return extensionClasses;
}
private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
String fileName = dir + type.getName();
try {
Enumeration<URL> urls;
if (classLoader != null) {
urls = classLoader.getResources(fileName);
} else {
urls = ClassLoader.getSystemResources(fileName);
}
if (urls != null) {
while (urls.hasMoreElements()) {
URL url = urls.nextElement();
try {
BufferedReader reader = null;
try {
reader = new BufferedReader(
new InputStreamReader(url.openStream(), StandardCharsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
final int ci = line.indexOf('#');
if (ci >= 0) line = line.substring(0, ci);
line = line.trim();
if (line.length() > 0) {
try {
String name = null;
int i = line.indexOf('=');
if (i > 0) {
name = line.substring(0, i).trim();
line = line.substring(i + 1).trim();
}
if (line.length() > 0) {
Class<?> clazz = classLoader.loadClass(line);
// Class<?> clazz =
// Class.forName(line, true,
// classLoader);
if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException(
"Error when load extension class(interface: " + type
+ ", class line: " + clazz.getName()
+ "), class " + clazz.getName()
+ "is not subtype of interface.");
} else {
try {
clazz.getConstructor(type);
} catch (NoSuchMethodException e) {
clazz.getConstructor();
String[] names = NAME_SEPARATOR.split(name);
if (names != null && names.length > 0) {
for (String n : names) {
if (!cachedNames.containsKey(clazz)) {
cachedNames.put(clazz, n);
}
Class<?> c = extensionClasses.get(n);
if (c == null) {
extensionClasses.put(n, clazz);
} else if (c != clazz) {
cachedNames.remove(clazz);
throw new IllegalStateException(
"Duplicate extension " + type.getName() + " name "
+ n + " on "
+ c.getName() + " and "
+ clazz.getName());
}
}
}
}
}
}
} catch (Throwable t) {
IllegalStateException e = new IllegalStateException(
"Failed to load extension class(interface: " + type + ", class line: "
+ line + ") in " + url
+ ", cause: "
+ t.getMessage(),
t);
exceptions.put(line, e);
}
}
} // end of while read lines
} finally {
if (reader != null) {
reader.close();
}
}
} catch (Throwable t) {
logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+ ") in " + url,
t);
}
} // end of while urls
}
} catch (Throwable t) {
logger.error(
"Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
t);
}
}
@SuppressWarnings("unused")
private static ClassLoader findClassLoader() {
return ExtensionLoader.class.getClassLoader();
}
@Override
public String toString() {
return this.getClass().getName() + "[" + type.getName() + "]";
}
private static class Holder<T> {
private volatile T value;
private void set(T value) {
this.value = value;
}
private T get() {
return value;
}
}
}

18
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java Normal file
View File

@@ -0,0 +1,18 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.lang.annotation.*;
/**
* SPI装载器注解
*
* @author rewerma @ 2018-10-20
* @version 1.0.0
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE })
public @interface SPI {
// Default SPI name
String value() default "";
}

88
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java Normal file
View File

@@ -0,0 +1,88 @@
package com.alibaba.otter.canal.connector.core.spi;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Enumeration;
import java.util.NoSuchElementException;
public class URLClassExtensionLoader extends URLClassLoader {
public URLClassExtensionLoader(URL[] urls) {
super(urls);
}
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
Class<?> c = findLoadedClass(name);
if (c != null) {
return c;
}
if (name.startsWith("java.") || name.startsWith("org.slf4j.")
|| name.startsWith("org.apache.logging")
|| name.startsWith("org.apache.commons.logging.")) {
// || name.startsWith("org.apache.hadoop."))
// {
c = super.loadClass(name);
}
if (c != null) return c;
try {
// 先加载jar内的class，可避免jar冲突
c = findClass(name);
} catch (ClassNotFoundException e) {
c = null;
}
if (c != null) {
return c;
}
return super.loadClass(name);
}
@Override
public Enumeration<URL> getResources(String name) throws IOException {
@SuppressWarnings("unchecked")
Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
tmp[0] = findResources(name); // local class
// path first
// tmp[1] = super.getResources(name);
return new CompoundEnumeration<>(tmp);
}
private static class CompoundEnumeration<E> implements Enumeration<E> {
private Enumeration<E>[] enums;
private int index = 0;
public CompoundEnumeration(Enumeration<E>[] enums){
this.enums = enums;
}
private boolean next() {
while (this.index < this.enums.length) {
if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
return true;
}
++this.index;
}
return false;
}
public boolean hasMoreElements() {
return this.next();
}
public E nextElement() {
if (!this.next()) {
throw new NoSuchElementException();
} else {
return this.enums[this.index].nextElement();
}
}
}
}

13
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java Normal file
View File

@@ -0,0 +1,13 @@
package com.alibaba.otter.canal.connector.core.util;
/**
* MQ 回调类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public interface Callback {
void commit();
void rollback();
}

65
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java → connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java
View File

@@ -1,20 +1,25 @@
package com.alibaba.otter.canal.common; package com.alibaba.otter.canal.connector.core.util;
import java.util.List; import java.util.List;
import org.apache.kafka.common.errors.SerializationException; import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket; import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream; import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat; import com.google.protobuf.WireFormat;
public class CanalMessageSerializer { /**
* Canal message 序列化工具类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class CanalMessageSerializerUtil {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
public static byte[] serializer(Message data, boolean filterTransactionEntry) { public static byte[] serializer(Message data, boolean filterTransactionEntry) {
@@ -58,7 +63,8 @@ public class CanalMessageSerializer {
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) { for (CanalEntry.Entry entry : data.getEntries()) {
if (filterTransactionEntry if (filterTransactionEntry
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) { && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
continue; continue;
} }
@@ -67,15 +73,60 @@ public class CanalMessageSerializer {
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES); packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setVersion(NettyUtils.VERSION); packetBuilder.setVersion(1);
packetBuilder.setBody(messageBuilder.build().toByteString()); packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray(); return packetBuilder.build().toByteArray();
} }
} }
} }
} catch (Exception e) { } catch (Exception e) {
throw new SerializationException("Error when serializing message to byte[] "); throw new RuntimeException("Error when serializing message to byte[] ");
} }
return null; return null;
} }
public static Message deserializer(byte[] data) {
return deserializer(data, false);
}
public static Message deserializer(byte[] data, boolean lazyParseEntry) {
try {
if (data == null) {
return null;
} else {
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
switch (p.getType()) {
case MESSAGES: {
if (!p.getCompression().equals(CanalPacket.Compression.NONE)
&& !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
throw new CanalClientException("compression is not supported in this connector");
}
CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
Message result = new Message(messages.getBatchId());
if (lazyParseEntry) {
// byteString
result.setRawEntries(messages.getMessagesList());
result.setRaw(true);
} else {
for (ByteString byteString : messages.getMessagesList()) {
result.addEntry(CanalEntry.Entry.parseFrom(byteString));
}
result.setRaw(false);
}
return result;
}
case ACK: {
CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
}
default: {
throw new CanalClientException("unexpected packet type: " + p.getType());
}
}
}
} catch (Exception e) {
throw new CanalClientException("deserializer failed", e);
}
}
} }

46
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java Normal file
View File

@@ -0,0 +1,46 @@
package com.alibaba.otter.canal.connector.core.util;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.util.Date;
public class DateUtil {
private static DateTimeZone dateTimeZone;
static {
dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE);
}
/**
* 通用日期时间字符解析
*
* @param datetimeStr 日期时间字符串
* @return Date
*/
public static Date parseDate(String datetimeStr) {
if (StringUtils.isEmpty(datetimeStr)) {
return null;
}
datetimeStr = datetimeStr.trim().replace('/', '-');
if (datetimeStr.contains("-")) {
if (datetimeStr.contains(":")) {
datetimeStr = datetimeStr.replace(" ", "T");
}
} else if (datetimeStr.contains(":")) {
datetimeStr = "T" + datetimeStr;
} else {
if (datetimeStr.length() == 8) {
String year = datetimeStr.substring(0, 4);
String month = datetimeStr.substring(4, 6);
String day = datetimeStr.substring(6, 8);
datetimeStr = year + "-" + month + "-" + day;
}
}
DateTime dateTime = new DateTime(datetimeStr, dateTimeZone);
return dateTime.toDate();
}
}

165
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java Normal file
View File

@@ -0,0 +1,165 @@
package com.alibaba.otter.canal.connector.core.util;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 类型转换工具类
*
* @author rewerma 2018-8-19 下午06:14:23
* @version 1.0.0
*/
public class JdbcTypeUtil {
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
return rs.getByte(columnName);
} else {
return rs.getObject(columnName);
}
}
public static Class<?> jdbcType2javaType(int jdbcType) {
switch (jdbcType) {
case Types.BIT:
case Types.BOOLEAN:
// return Boolean.class;
case Types.TINYINT:
return Byte.TYPE;
case Types.SMALLINT:
return Short.class;
case Types.INTEGER:
return Integer.class;
case Types.BIGINT:
return Long.class;
case Types.DECIMAL:
case Types.NUMERIC:
return BigDecimal.class;
case Types.REAL:
return Float.class;
case Types.FLOAT:
case Types.DOUBLE:
return Double.class;
case Types.CHAR:
case Types.VARCHAR:
case Types.LONGVARCHAR:
return String.class;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
return byte[].class;
case Types.DATE:
return Date.class;
case Types.TIME:
return Time.class;
case Types.TIMESTAMP:
return Timestamp.class;
default:
return String.class;
}
}
private static boolean isText(String columnType) {
return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
|| "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
}
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR
|| sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
return null;
}
try {
Object res;
switch (sqlType) {
case Types.INTEGER:
res = Integer.parseInt(value);
break;
case Types.SMALLINT:
res = Short.parseShort(value);
break;
case Types.BIT:
case Types.TINYINT:
res = Byte.parseByte(value);
break;
case Types.BIGINT:
if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
res = new BigInteger(value);
} else {
res = Long.parseLong(value);
}
break;
// case Types.BIT:
case Types.BOOLEAN:
res = !"0".equals(value);
break;
case Types.DOUBLE:
case Types.FLOAT:
res = Double.parseDouble(value);
break;
case Types.REAL:
res = Float.parseFloat(value);
break;
case Types.DECIMAL:
case Types.NUMERIC:
res = new BigDecimal(value);
break;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
res = value.getBytes("ISO-8859-1");
break;
case Types.DATE:
if (!value.startsWith("0000-00-00")) {
java.util.Date date = DateUtil.parseDate(value);
if (date != null) {
res = new Date(date.getTime());
} else {
res = null;
}
} else {
res = null;
}
break;
case Types.TIME: {
java.util.Date date = DateUtil.parseDate(value);
if (date != null) {
res = new Time(date.getTime());
} else {
res = null;
}
break;
}
case Types.TIMESTAMP:
if (!value.startsWith("0000-00-00")) {
java.util.Date date = DateUtil.parseDate(value);
if (date != null) {
res = new Timestamp(date.getTime());
} else {
res = null;
}
} else {
res = null;
}
break;
case Types.CLOB:
default:
res = value;
break;
}
return res;
} catch (Exception e) {
logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
return value;
}
}
}

131
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java Normal file
View File

@@ -0,0 +1,131 @@
package com.alibaba.otter.canal.connector.core.util;
import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
/**
* Message对象解析工具类
*
* @author rewerma 2018-8-19 下午06:14:23
* @version 1.0.0
*/
public class MessageUtil {
public static List<CommonMessage> convert(Message message) {
if (message == null) {
return null;
}
List<CanalEntry.Entry> entries = message.getEntries();
List<CommonMessage> msgs = new ArrayList<>(entries.size());
for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e);
}
CanalEntry.EventType eventType = rowChange.getEventType();
final CommonMessage msg = new CommonMessage();
msg.setIsDdl(rowChange.getIsDdl());
msg.setDatabase(entry.getHeader().getSchemaName());
msg.setTable(entry.getHeader().getTableName());
msg.setType(eventType.toString());
msg.setEs(entry.getHeader().getExecuteTime());
msg.setIsDdl(rowChange.getIsDdl());
msg.setTs(System.currentTimeMillis());
msg.setSql(rowChange.getSql());
msgs.add(msg);
List<Map<String, Object>> data = new ArrayList<>();
List<Map<String, Object>> old = new ArrayList<>();
if (!rowChange.getIsDdl()) {
Set<String> updateSet = new HashSet<>();
msg.setPkNames(new ArrayList<>());
int i = 0;
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
List<CanalEntry.Column> columns;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
for (CanalEntry.Column column : columns) {
if (i == 0) {
if (column.getIsKey()) {
msg.getPkNames().add(column.getName());
}
}
if (column.getIsNull()) {
row.put(column.getName(), null);
} else {
row.put(column.getName(),
JdbcTypeUtil.typeConvert(msg.getTable(),
column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
}
// 获取update为true的字段
if (column.getUpdated()) {
updateSet.add(column.getName());
}
}
if (!row.isEmpty()) {
data.add(row);
}
if (eventType == CanalEntry.EventType.UPDATE) {
Map<String, Object> rowOld = new LinkedHashMap<>();
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
if (column.getIsNull()) {
rowOld.put(column.getName(), null);
} else {
rowOld.put(column.getName(),
JdbcTypeUtil.typeConvert(msg.getTable(),
column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
}
}
}
// update操作将记录修改前的值
if (!rowOld.isEmpty()) {
old.add(rowOld);
}
}
i++;
}
if (!data.isEmpty()) {
msg.setData(data);
}
if (!old.isEmpty()) {
msg.setOld(old);
}
}
}
return msgs;
}
}

22
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java Normal file
View File

@@ -0,0 +1,22 @@
package com.alibaba.otter.canal.connector.core.util;
public class TimeZone {
public final static String LOCATION_TIME_ZONE; // 当前时区
static {
java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault();
int rawOffset = localTimeZone.getRawOffset();
String symbol = "+";
if (rawOffset < 0) {
symbol = "-";
}
rawOffset = Math.abs(rawOffset);
int offsetHour = rawOffset / 3600000;
int offsetMinute = rawOffset % 3600000 / 60000;
String hour = String.format("%1$02d", offsetHour);
String minute = String.format("%1$02d", offsetMinute);
LOCATION_TIME_ZONE = symbol + hour + ":" + minute;
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("GMT" + LOCATION_TIME_ZONE));
}
}

94
connector/kafka-connector/pom.xml Normal file
View File

@@ -0,0 +1,94 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.kafka</artifactId>
<packaging>jar</packaging>
<name>canal connector kafka module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

16
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java Normal file
View File

@@ -0,0 +1,16 @@
package com.alibaba.otter.canal.connector.kafka.config;
/**
* Kafka 配置常量类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class KafkaConstants {
public static final String ROOT = "canal";
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE = ROOT + "." + "mq.kafka.kerberos.krb5.file";
public static final String CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE = ROOT + "." + "mq.kafka.kerberos.jaas.file";
}

53
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java Normal file
View File

@@ -0,0 +1,53 @@
package com.alibaba.otter.canal.connector.kafka.config;
import java.util.LinkedHashMap;
import java.util.Map;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* Kafka producer 配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class KafkaProducerConfig extends MQProperties {
private Map<String, Object> kafkaProperties = new LinkedHashMap<>();
private boolean kerberosEnabled = false;
private String krb5File;
private String jaasFile;
public Map<String, Object> getKafkaProperties() {
return kafkaProperties;
}
public void setKafkaProperties(Map<String, Object> kafkaProperties) {
this.kafkaProperties = kafkaProperties;
}
public boolean isKerberosEnabled() {
return kerberosEnabled;
}
public void setKerberosEnabled(boolean kerberosEnabled) {
this.kerberosEnabled = kerberosEnabled;
}
public String getKrb5File() {
return krb5File;
}
public void setKrb5File(String krb5File) {
this.krb5File = krb5File;
}
public String getJaasFile() {
return jaasFile;
}
public void setJaasFile(String jaasFile) {
this.jaasFile = jaasFile;
}
}

137
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java Normal file
View File

@@ -0,0 +1,137 @@
package com.alibaba.otter.canal.connector.kafka.consumer;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka consumer SPI 实现
*
* @author rewerma @ 2020-02-01
* @version 1.0.0
*/
@SPI("kafka")
public class CanalKafkaConsumer implements CanalMsgConsumer {
private static final String PREFIX_KAFKA_CONFIG = "kafka.";
private KafkaConsumer<String, ?> kafkaConsumer;
private boolean flatMessage = true;
private String topic;
private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
private Properties kafkaProperties = new Properties();
@Override
public void init(Properties properties, String topic, String groupId) {
this.topic = topic;
Boolean flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
if (flatMessage != null) {
this.flatMessage = flatMessage;
}
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
String k = (String) entry.getKey();
Object v = entry.getValue();
if (k.startsWith(PREFIX_KAFKA_CONFIG) && v != null) {
kafkaProperties.put(k.substring(PREFIX_KAFKA_CONFIG.length()), v);
}
}
kafkaProperties.put("group.id", groupId);
kafkaProperties.put("key.deserializer", StringDeserializer.class);
kafkaProperties.put("client.id", UUID.randomUUID().toString().substring(0, 6));
}
@Override
public void connect() {
if (this.flatMessage) {
kafkaProperties.put("value.deserializer", StringDeserializer.class);
this.kafkaConsumer = new KafkaConsumer<String, String>(kafkaProperties);
} else {
kafkaProperties.put("value.deserializer", KafkaMessageDeserializer.class);
this.kafkaConsumer = new KafkaConsumer<String, Message>(kafkaProperties);
}
kafkaConsumer.subscribe(Collections.singletonList(topic));
}
@SuppressWarnings("unchecked")
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
if (!flatMessage) {
ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer
.poll(unit.toMillis(timeout));
if (!records.isEmpty()) {
currentOffsets.clear();
List<CommonMessage> messages = new ArrayList<>();
for (ConsumerRecord<String, Message> record : records) {
if (currentOffsets.get(record.partition()) == null) {
currentOffsets.put(record.partition(), record.offset());
}
messages.addAll(MessageUtil.convert(record.value()));
}
return messages;
}
} else {
ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer
.poll(unit.toMillis(timeout));
if (!records.isEmpty()) {
List<CommonMessage> messages = new ArrayList<>();
currentOffsets.clear();
for (ConsumerRecord<String, String> record : records) {
if (currentOffsets.get(record.partition()) == null) {
currentOffsets.put(record.partition(), record.offset());
}
String flatMessageJson = record.value();
CommonMessage flatMessages = JSON.parseObject(flatMessageJson, CommonMessage.class);
messages.add(flatMessages);
}
return messages;
}
}
return null;
}
@Override
public void rollback() {
// 回滚所有分区
if (kafkaConsumer != null) {
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey()));
kafkaConsumer.commitSync();
}
}
}
@Override
public void ack() {
if (kafkaConsumer != null) {
kafkaConsumer.commitSync();
}
}
@Override
public void disconnect() {
if (kafkaConsumer != null) {
kafkaConsumer.unsubscribe();
}
if (kafkaConsumer != null) {
kafkaConsumer.close();
kafkaConsumer = null;
}
}
}

31
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java Normal file
View File

@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.connector.kafka.consumer;
import java.util.Map;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import org.apache.kafka.common.serialization.Deserializer;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka Message类的反序列化
*
* @author rewerma @ 2018-6-12
* @version 1.0.0
*/
public class KafkaMessageDeserializer implements Deserializer<Message> {
@Override
public void configure(Map<String, ?> configs, boolean isKey) {
}
@Override
public Message deserialize(String topic1, byte[] data) {
return CanalMessageSerializerUtil.deserializer(data);
}
@Override
public void close() {
// nothing to do
}
}

178
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java → connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java
View File

@@ -1,14 +1,12 @@
package com.alibaba.otter.canal.kafka; package com.alibaba.otter.canal.connector.kafka.producer;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.Producer;
@@ -19,68 +17,90 @@ import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate; import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/** /**
* kafka producer 主操作类 * kafka producer SPI 实现
* *
* @author machengyuan 2018-6-11 下午05:30:49 * @author rewerma 2018-6-11 下午05:30:49
* @version 1.0.0 * @version 1.0.0
*/ */
@SuppressWarnings({ "rawtypes", "unchecked" })
@SPI("kafka")
public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer { public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class); private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
private static final String PREFIX_KAFKA_CONFIG = "kafka.";
private Producer<String, byte[]> producer; private Producer<String, byte[]> producer;
private MQProperties kafkaProperties;
@Override @Override
public void init(MQProperties kafkaProperties) { public void init(Properties properties) {
super.init(kafkaProperties); KafkaProducerConfig kafkaProducerConfig = new KafkaProducerConfig();
this.mqProperties = kafkaProducerConfig;
super.init(properties);
// load properties
this.loadKafkaProperties(properties);
this.kafkaProperties = kafkaProperties; Properties kafkaProperties = new Properties();
Properties properties = new Properties(); kafkaProperties.putAll(kafkaProducerConfig.getKafkaProperties());
properties.put("bootstrap.servers", kafkaProperties.getServers()); kafkaProperties.put("key.serializer", StringSerializer.class);
properties.put("acks", kafkaProperties.getAcks()); if (kafkaProducerConfig.isKerberosEnabled()) {
properties.put("compression.type", kafkaProperties.getCompressionType()); File krb5File = new File(kafkaProducerConfig.getKrb5File());
properties.put("batch.size", kafkaProperties.getBatchSize()); File jaasFile = new File(kafkaProducerConfig.getJaasFile());
properties.put("linger.ms", kafkaProperties.getLingerMs());
properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
properties.put("buffer.memory", kafkaProperties.getBufferMemory());
properties.put("key.serializer", StringSerializer.class.getName());
properties.put("max.in.flight.requests.per.connection", 1);
if (!kafkaProperties.getProperties().isEmpty()) {
properties.putAll(kafkaProperties.getProperties());
}
properties.put("retries", kafkaProperties.getRetries());
if (kafkaProperties.isKerberosEnable()) {
File krb5File = new File(kafkaProperties.getKerberosKrb5FilePath());
File jaasFile = new File(kafkaProperties.getKerberosJaasFilePath());
if (krb5File.exists() && jaasFile.exists()) { if (krb5File.exists() && jaasFile.exists()) {
// 配置kerberos认证，需要使用绝对路径 // 配置kerberos认证，需要使用绝对路径
System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath()); System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath());
System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath()); System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath());
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
properties.put("security.protocol", "SASL_PLAINTEXT"); kafkaProperties.put("security.protocol", "SASL_PLAINTEXT");
properties.put("sasl.kerberos.service.name", "kafka"); kafkaProperties.put("sasl.kerberos.service.name", "kafka");
} else { } else {
String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it"; String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it";
logger.error(errorMsg); logger.error(errorMsg);
throw new RuntimeException(errorMsg); throw new RuntimeException(errorMsg);
} }
} }
kafkaProperties.put("value.serializer", KafkaMessageSerializer.class);
producer = new KafkaProducer<>(kafkaProperties);
}
properties.put("value.serializer", KafkaMessageSerializer.class.getName()); private void loadKafkaProperties(Properties properties) {
producer = new KafkaProducer<String, byte[]>(properties); KafkaProducerConfig kafkaProducerConfig = (KafkaProducerConfig) this.mqProperties;
Map<String, Object> kafkaProperties = kafkaProducerConfig.getKafkaProperties();
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
String key = (String) entry.getKey();
Object value = entry.getValue();
if (key.startsWith(PREFIX_KAFKA_CONFIG) && value != null) {
key = key.substring(PREFIX_KAFKA_CONFIG.length());
kafkaProperties.put(key, value);
}
}
String kerberosEnabled = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
if (!StringUtils.isEmpty(kerberosEnabled)) {
kafkaProducerConfig.setKerberosEnabled(Boolean.parseBoolean(kerberosEnabled));
}
String krb5File = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE);
if (!StringUtils.isEmpty(krb5File)) {
kafkaProducerConfig.setKrb5File(krb5File);
}
String jaasFile = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE);
if (!StringUtils.isEmpty(jaasFile)) {
kafkaProducerConfig.setJaasFile(jaasFile);
}
} }
@Override @Override
@@ -90,41 +110,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
if (producer != null) { if (producer != null) {
producer.close(); producer.close();
} }
super.stop();
} catch (Throwable e) { } catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka producer:", e); logger.warn("##something goes wrong when stopping kafka producer:", e);
} finally { } finally {
logger.info("## kafka producer is down."); logger.info("## kafka producer is down.");
} }
super.stop();
} }
@Override @Override
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) { public void send(MQDestination mqDestination, Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor); ExecutorTemplate template = new ExecutorTemplate(executor);
boolean flat = kafkaProperties.getFlatMessage();
try { try {
List result = null; List result;
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) { if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) {
// 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化 // 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
Map<String, Message> messageMap = MQMessageUtils.messageTopics(message, Map<String, Message> messageMap = MQMessageUtils
canalDestination.getTopic(), .messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic());
canalDestination.getDynamicTopic());
// 针对不同的topic,引入多线程提升效率 // 针对不同的topic,引入多线程提升效率
for (Map.Entry<String, Message> entry : messageMap.entrySet()) { for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_'); final String topicName = entry.getKey().replace('.', '_');
final Message messageSub = entry.getValue(); final Message messageSub = entry.getValue();
template.submit(new Callable() { template.submit((Callable) () -> {
try {
@Override return send(mqDestination, topicName, messageSub, mqProperties.isFlatMessage());
public List<Future> call() throws Exception { } catch (Exception e) {
try { throw new RuntimeException(e);
return send(canalDestination, topicName, messageSub, flat);
} catch (Exception e) {
throw new RuntimeException(e);
}
} }
}); });
} }
@@ -132,7 +145,10 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
result = template.waitForResult(); result = template.waitForResult();
} else { } else {
result = new ArrayList(); result = new ArrayList();
List<Future> futures = send(canalDestination, canalDestination.getTopic(), message, flat); List<Future> futures = send(mqDestination,
mqDestination.getTopic(),
message,
mqProperties.isFlatMessage());
result.add(futures); result.add(futures);
} }
@@ -161,36 +177,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
} }
} }
private List<Future> send(MQProperties.CanalDestination canalDestination, String topicName, Message message, private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
boolean flat) throws Exception { List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
List<ProducerRecord<String, byte[]>> records = new ArrayList<ProducerRecord<String, byte[]>>();
if (!flat) { if (!flat) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) { if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
// 并发构造 // 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor); EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区 // 串行分区
Message[] messages = MQMessageUtils.messagePartition(datas, Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(), message.getId(),
canalDestination.getPartitionsNum(), mqDestination.getPartitionsNum(),
canalDestination.getPartitionHash(), mqDestination.getPartitionHash(),
kafkaProperties.getDatabaseHash()); this.mqProperties.isDatabaseHash());
int length = messages.length; int length = messages.length;
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
Message messagePartition = messages[i]; Message messagePartition = messages[i];
if (messagePartition != null) { if (messagePartition != null) {
records.add(new ProducerRecord<String, byte[]>(topicName, records.add(new ProducerRecord<>(topicName,
i, i,
null, null,
CanalMessageSerializer.serializer(messagePartition, CanalMessageSerializerUtil.serializer(messagePartition, true)));
kafkaProperties.isFilterTransactionEntry())));
} }
} }
} else { } else {
final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0; final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
records.add(new ProducerRecord<String, byte[]>(topicName, records.add(new ProducerRecord<>(topicName,
partition, partition,
null, null,
CanalMessageSerializer.serializer(message, kafkaProperties.isFilterTransactionEntry()))); CanalMessageSerializerUtil.serializer(message, true)));
} }
} else { } else {
// 发送扁平数据json // 发送扁平数据json
@@ -199,24 +213,24 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
// 串行分区 // 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId()); List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
for (FlatMessage flatMessage : flatMessages) { for (FlatMessage flatMessage : flatMessages) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) { if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
canalDestination.getPartitionsNum(), mqDestination.getPartitionsNum(),
canalDestination.getPartitionHash(), mqDestination.getPartitionHash(),
kafkaProperties.getDatabaseHash()); this.mqProperties.isDatabaseHash());
int length = partitionFlatMessage.length; int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i]; FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) { if (flatMessagePart != null) {
records.add(new ProducerRecord<String, byte[]>(topicName, records.add(new ProducerRecord<>(topicName,
i, i,
null, null,
JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue))); JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
} }
} }
} else { } else {
final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0; final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
records.add(new ProducerRecord<String, byte[]>(topicName, records.add(new ProducerRecord<>(topicName,
partition, partition,
null, null,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))); JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
@@ -224,11 +238,11 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
} }
} }
return produce(topicName, records, flat); return produce(records);
} }
private List<Future> produce(String topicName, List<ProducerRecord<String, byte[]>> records, boolean flatMessage) { private List<Future> produce(List<ProducerRecord<String, byte[]>> records) {
List<Future> futures = new ArrayList<Future>(); List<Future> futures = new ArrayList<>();
// 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题 // 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题
for (ProducerRecord record : records) { for (ProducerRecord record : records) {
futures.add(producer.send(record)); futures.add(producer.send(record));

8
server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java → connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java
View File

@@ -1,13 +1,13 @@
package com.alibaba.otter.canal.kafka; package com.alibaba.otter.canal.connector.kafka.producer;
import java.util.Map;
import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.Serializer;
import java.util.Map;
/** /**
* Kafka Message类的序列化 * Kafka Message类的序列化
* *
* @author machengyuan 2018-6-11 下午05:30:49 * @author rewerma 2018-6-11 下午05:30:49
* @version 1.0.0 * @version 1.0.0
*/ */
public class KafkaMessageSerializer implements Serializer<byte[]> { public class KafkaMessageSerializer implements Serializer<byte[]> {

1
connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer Normal file
View File

@@ -0,0 +1 @@
kafka=com.alibaba.otter.canal.connector.kafka.producer.CanalKafkaProducer

1
connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer Normal file
View File

@@ -0,0 +1 @@
kafka=com.alibaba.otter.canal.connector.kafka.consumer.CanalKafkaConsumer

29
connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java Normal file
View File

@@ -0,0 +1,29 @@
package com.alibaba.otter.canal.connector.kafka.test;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Properties;
import org.junit.Ignore;
import org.junit.Test;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
@Ignore
public class CanalKafkaProducerTest {
@Test
public void testLoadKafkaProducer() throws IOException {
Properties pro = new Properties();
FileInputStream in = new FileInputStream("../../deployer/src/main/resources/canal.properties");
pro.load(in);
ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
CanalMQProducer canalMQProducer = loader
.getExtension("kafka", "/../../deployer/target/canal/plugin", "/../../deployer/target/canal/plugin");
canalMQProducer.init(pro);
in.close();
}
}

131
connector/pom.xml Normal file
View File

@@ -0,0 +1,131 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.connector</artifactId>
<version>1.1.5-SNAPSHOT</version>
<packaging>pom</packaging>
<name>canal connector module for otter ${project.version}</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.test.skip>true</maven.test.skip>
<downloadSources>true</downloadSources>
<java_source_version>1.8</java_source_version>
<java_target_version>1.8</java_target_version>
<file_encoding>UTF-8</file_encoding>
</properties>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
</license>
</licenses>
<scm>
<url>git@github.com:alibaba/canal.git</url>
<connection>scm:git:git@github.com:alibaba/canal.git</connection>
<developerConnection>scm:git:git@github.com:alibaba/canal.git</developerConnection>
</scm>
<repositories>
<repository>
<id>central</id>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>java.net</id>
<url>http://download.java.net/maven/2/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>aliyun</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype</id>
<name>sonatype</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype-release</id>
<name>sonatype-release</name>
<url>https://oss.sonatype.org/service/local/repositories/releases/content</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<modules>
<module>kafka-connector</module>
<module>core</module>
<module>rocketmq-connector</module>
<module>rabbitmq-connector</module>
<module>tcp-connector</module>
</modules>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>${java_source_version}</source>
<target>${java_target_version}</target>
<encoding>${file_encoding}</encoding>
</configuration>
</plugin>
</plugins>
</build>
<distributionManagement>
<snapshotRepository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>sonatype-nexus-staging</id>
<name>Nexus Release Repository</name>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
</project>

99
connector/rabbitmq-connector/pom.xml Normal file
View File

@@ -0,0 +1,99 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<packaging>jar</packaging>
<name>canal connector rabbitmq module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>5.5.0</version>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
<version>1.0.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

20
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java Normal file
View File

@@ -0,0 +1,20 @@
package com.alibaba.otter.canal.connector.rabbitmq.config;
/**
* RabbitMQ 配置常量类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RabbitMQConstants {
public static final String ROOT = "rabbitmq";
public static final String RABBITMQ_HOST = ROOT + "." + "host";
public static final String RABBITMQ_EXCHANGE = ROOT + "." + "exchange";
public static final String RABBITMQ_VIRTUAL_HOST = ROOT + "." + "virtual.host";
public static final String RABBITMQ_USERNAME = ROOT + "." + "username";
public static final String RABBITMQ_PASSWORD = ROOT + "." + "password";
public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId";
}

58
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java Normal file
View File

@@ -0,0 +1,58 @@
package com.alibaba.otter.canal.connector.rabbitmq.config;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* RabbitMQ 配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RabbitMQProducerConfig extends MQProperties {
private String host;
private String virtualHost;
private String exchange;
private String username;
private String password;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getVirtualHost() {
return virtualHost;
}
public void setVirtualHost(String virtualHost) {
this.virtualHost = virtualHost;
}
public String getExchange() {
return exchange;
}
public void setExchange(String exchange) {
this.exchange = exchange;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}

214
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java Normal file
View File

@@ -0,0 +1,214 @@
package com.alibaba.otter.canal.connector.rabbitmq.consumer;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.common.collect.Lists;
import com.rabbitmq.client.*;
/**
* RabbitMQ consumer SPI 实现
*
* @author rewerma 2020-02-01
* @version 1.0.0
*/
@SPI("rabbitmq")
public class CanalRabbitMQConsumer implements CanalMsgConsumer {
private static final Logger logger = LoggerFactory
.getLogger(CanalRabbitMQConsumer.class);
// 链接地址
private String nameServer;
// 主机名
private String vhost;
private String queueName;
// 一些鉴权信息
private String accessKey;
private String secretKey;
private Long resourceOwnerId;
private String username;
private String password;
private boolean flatMessage;
private Connection connect;
private Channel channel;
private long batchProcessTimeout = 60 * 1000;
private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
private volatile ConsumerBatchMessage<CommonMessage> lastGetBatchMessage = null;
@Override
public void init(Properties properties, String topic, String groupId) {
this.nameServer = properties.getProperty("rabbitmq.host");
this.vhost = properties.getProperty("rabbitmq.virtual.host");
this.queueName = topic;
this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
this.username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
this.password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
Long resourceOwnerIdPro = (Long) properties.get(RabbitMQConstants.RABBITMQ_RESOURCE_OWNERID);
if (resourceOwnerIdPro != null) {
this.resourceOwnerId = resourceOwnerIdPro;
}
this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
}
@Override
public void connect() {
ConnectionFactory factory = new ConnectionFactory();
if (accessKey.length() > 0 && secretKey.length() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(accessKey, secretKey, resourceOwnerId));
} else {
factory.setUsername(username);
factory.setPassword(password);
}
factory.setHost(nameServer);
factory.setAutomaticRecoveryEnabled(true);
factory.setNetworkRecoveryInterval(5000);
factory.setVirtualHost(vhost);
try {
connect = factory.newConnection();
channel = connect.createChannel();
} catch (IOException | TimeoutException e) {
throw new CanalClientException("Start RabbitMQ producer error", e);
}
// 不存在连接 则重新连接
if (connect == null) {
this.connect();
}
Consumer consumer = new DefaultConsumer(channel) {
@Override
public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties,
byte[] body) throws IOException {
if (body != null) {
channel.basicAck(envelope.getDeliveryTag(), process(body));
}
}
};
try {
channel.basicConsume(queueName, false, consumer);
} catch (IOException e) {
throw new CanalClientException("error", e);
}
}
private boolean process(byte[] messageData) {
if (logger.isDebugEnabled()) {
logger.debug("Get Message: {}", new String(messageData));
}
List<CommonMessage> messageList = Lists.newArrayList();
if (!flatMessage) {
Message message = CanalMessageSerializerUtil.deserializer(messageData);
messageList.addAll(MessageUtil.convert(message));
} else {
CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
messageList.add(commonMessage);
}
ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
try {
messageBlockingQueue.put(batchMessage);
} catch (InterruptedException e) {
logger.error("Put message to queue error", e);
throw new RuntimeException(e);
}
boolean isCompleted;
try {
isCompleted = batchMessage.waitFinish(batchProcessTimeout);
} catch (InterruptedException e) {
logger.error("Interrupted when waiting messages to be finished.", e);
throw new RuntimeException(e);
}
boolean isSuccess = batchMessage.isSuccess();
return isCompleted && isSuccess;
}
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
try {
if (this.lastGetBatchMessage != null) {
throw new CanalClientException("mq get/ack not support concurrent & async ack");
}
ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
if (batchMessage != null) {
this.lastGetBatchMessage = batchMessage;
return batchMessage.getData();
}
} catch (InterruptedException ex) {
logger.warn("Get message timeout", ex);
throw new CanalClientException("Failed to fetch the data after: " + timeout);
}
return null;
}
@Override
public void rollback() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void ack() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.ack();
}
} catch (Throwable e) {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void disconnect() {
if (connect != null) {
try {
connect.close();
} catch (IOException e) {
throw new CanalClientException("stop connect error", e);
}
}
if (channel != null) {
try {
channel.close();
} catch (IOException | TimeoutException e) {
throw new CanalClientException("stop channel error", e);
}
}
}
}

48
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java Normal file
View File

@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.connector.rabbitmq.consumer;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
public class ConsumerBatchMessage<T> {
private final List<T> data;
private CountDownLatch latch;
private boolean hasFailure = false;
public ConsumerBatchMessage(List<T> data){
this.data = data;
latch = new CountDownLatch(1);
}
public boolean waitFinish(long timeout) throws InterruptedException {
return latch.await(timeout, TimeUnit.MILLISECONDS);
}
public boolean isSuccess() {
return !hasFailure;
}
public List<T> getData() {
return data;
}
/**
* Countdown if the sub message is successful.
*/
public void ack() {
latch.countDown();
}
/**
* Countdown and fail-fast if the sub message is failed.
*/
public void fail() {
hasFailure = true;
// fail fast
long count = latch.getCount();
for (int i = 0; i < count; i++) {
latch.countDown();
}
}
}

12
server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java → connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java
View File

@@ -1,8 +1,4 @@
/** package com.alibaba.otter.canal.connector.rabbitmq.producer;
* aliyun amqp协议 账号类
* 暂不支持STS授权情况
*/
package com.alibaba.otter.canal.rabbitmq;
import java.security.InvalidKeyException; import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
@@ -10,6 +6,12 @@ import java.security.NoSuchAlgorithmException;
import com.alibaba.mq.amqp.utils.UserUtils; import com.alibaba.mq.amqp.utils.UserUtils;
import com.rabbitmq.client.impl.CredentialsProvider; import com.rabbitmq.client.impl.CredentialsProvider;
/**
* aliyun amqp协议 账号类
* 暂不支持STS授权情况
*
* @version 1.0.0
*/
public class AliyunCredentialsProvider implements CredentialsProvider { public class AliyunCredentialsProvider implements CredentialsProvider {
/** /**

180
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java Normal file
View File

@@ -0,0 +1,180 @@
package com.alibaba.otter.canal.connector.rabbitmq.producer;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeoutException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
/**
* RabbitMQ Producer SPI 实现
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
@SPI("rabbitmq")
public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
private Connection connect;
private Channel channel;
@Override
public void init(Properties properties) {
RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig();
this.mqProperties = rabbitMQProperties;
super.init(properties);
loadRabbitMQProperties(properties);
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(rabbitMQProperties.getHost());
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
&& mqProperties.getAliyunUid() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
mqProperties.getAliyunSecretKey(),
mqProperties.getAliyunUid()));
} else {
factory.setUsername(rabbitMQProperties.getUsername());
factory.setPassword(rabbitMQProperties.getPassword());
}
factory.setVirtualHost(rabbitMQProperties.getVirtualHost());
try {
connect = factory.newConnection();
channel = connect.createChannel();
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
} catch (IOException | TimeoutException ex) {
throw new CanalException("Start RabbitMQ producer error", ex);
}
}
private void loadRabbitMQProperties(Properties properties) {
RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
String host = properties.getProperty(RabbitMQConstants.RABBITMQ_HOST);
if (!StringUtils.isEmpty(host)) {
rabbitMQProperties.setHost(host);
}
String vhost = properties.getProperty(RabbitMQConstants.RABBITMQ_VIRTUAL_HOST);
if (!StringUtils.isEmpty(vhost)) {
rabbitMQProperties.setVirtualHost(vhost);
}
String exchange = properties.getProperty(RabbitMQConstants.RABBITMQ_EXCHANGE);
if (!StringUtils.isEmpty(exchange)) {
rabbitMQProperties.setExchange(exchange);
}
String username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
if (!StringUtils.isEmpty(username)) {
rabbitMQProperties.setUsername(username);
}
String password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
if (!StringUtils.isEmpty(password)) {
rabbitMQProperties.setPassword(password);
}
}
@Override
public void send(final MQDestination destination, Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, Message> messageMap = MQMessageUtils
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
send(destination, topicName, messageSub);
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
private void send(MQDestination canalDestination, String topicName, Message messageSub) {
if (!mqProperties.isFlatMessage()) {
byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry());
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
} else {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
for (FlatMessage flatMessage : flatMessages) {
byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
}
}
}
private void sendMessage(String queueName, byte[] message) {
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
try {
RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void stop() {
logger.info("## Stop RabbitMQ producer##");
try {
this.connect.close();
this.channel.close();
super.stop();
} catch (IOException | TimeoutException ex) {
throw new CanalException("Stop RabbitMQ producer error", ex);
}
super.stop();
}
}

1
connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer Normal file
View File

@@ -0,0 +1 @@
rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.producer.CanalRabbitMQProducer

1
connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer Normal file
View File

@@ -0,0 +1 @@
rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.consumer.CanalRabbitMQConsumer

104
connector/rocketmq-connector/pom.xml Normal file
View File

@@ -0,0 +1,104 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rocketmq</artifactId>
<packaging>jar</packaging>
<name>canal connector rocketMQ module for otter ${project.version}</name>
<properties>
<rocketmq_version>4.5.2</rocketmq_version>
</properties>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar"/>
</fileset>
</copy>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar" />
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

25
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java Normal file
View File

@@ -0,0 +1,25 @@
package com.alibaba.otter.canal.connector.rocketmq.config;
/**
* RocketMQ 配置常量类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RocketMQConstants {
public static final String ROOT = "rocketmq";
public static final String ROCKETMQ_PRODUCER_GROUP = ROOT + "." + "producer.group";
public static final String ROCKETMQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "enable.message.trace";
public static final String ROCKETMQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "customized.trace.topic";
public static final String ROCKETMQ_NAMESPACE = ROOT + "." + "namespace";
public static final String ROCKETMQ_NAMESRV_ADDR = ROOT + "." + "namesrv.addr";
public static final String ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED = ROOT + "." + "retry.times.when.send.failed";
public static final String ROCKETMQ_VIP_CHANNEL_ENABLED = ROOT + "." + "vip.channel.enabled";
public static final String ROCKETMQ_ACCESS_CHANNEL = ROOT + "." + "access.channel";
public static final String ROCKETMQ_BATCH_SIZE = ROOT + "." + "batch.size";
public static final String ROCKETMQ_SUBSCRIBE_FILTER = ROOT + "." + "subscribe.filter";
}

76
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java Normal file
View File

@@ -0,0 +1,76 @@
package com.alibaba.otter.canal.connector.rocketmq.config;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* RocketMQ 配置类
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
public class RocketMQProducerConfig extends MQProperties {
private String producerGroup;
private boolean enableMessageTrace = false;
private String customizedTraceTopic;
private String namespace;
private String namesrvAddr;
private int retryTimesWhenSendFailed = 0;
private boolean vipChannelEnabled = false;
public String getProducerGroup() {
return producerGroup;
}
public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup;
}
public boolean isEnableMessageTrace() {
return enableMessageTrace;
}
public void setEnableMessageTrace(boolean enableMessageTrace) {
this.enableMessageTrace = enableMessageTrace;
}
public String getCustomizedTraceTopic() {
return customizedTraceTopic;
}
public void setCustomizedTraceTopic(String customizedTraceTopic) {
this.customizedTraceTopic = customizedTraceTopic;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
public String getNamesrvAddr() {
return namesrvAddr;
}
public void setNamesrvAddr(String namesrvAddr) {
this.namesrvAddr = namesrvAddr;
}
public int getRetryTimesWhenSendFailed() {
return retryTimesWhenSendFailed;
}
public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) {
this.retryTimesWhenSendFailed = retryTimesWhenSendFailed;
}
public boolean isVipChannelEnabled() {
return vipChannelEnabled;
}
public void setVipChannelEnabled(boolean vipChannelEnabled) {
this.vipChannelEnabled = vipChannelEnabled;
}
}

237
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java Normal file
View File

@@ -0,0 +1,237 @@
package com.alibaba.otter.canal.connector.rocketmq.consumer;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.common.collect.Lists;
/**
* RocketMQ consumer SPI 实现
*
* @author rewerma 2020-02-01
* @version 1.0.0
*/
@SPI("rocketmq")
public class CanalRocketMQConsumer implements CanalMsgConsumer {
private static final Logger logger = LoggerFactory
.getLogger(CanalRocketMQConsumer.class);
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
private String nameServer;
private String topic;
private String groupName;
private volatile boolean connected = false;
private DefaultMQPushConsumer rocketMQConsumer;
private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
private int batchSize = -1;
private long batchProcessTimeout = 60 * 1000;
private boolean flatMessage;
private volatile ConsumerBatchMessage<CommonMessage> lastGetBatchMessage = null;
private String accessKey;
private String secretKey;
private String customizedTraceTopic;
private boolean enableMessageTrace = false;
private String accessChannel;
private String namespace;
private String filter = "*";
@Override
public void init(Properties properties, String topic, String groupName) {
this.topic = topic;
this.groupName = groupName;
this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
if (StringUtils.isNotEmpty(enableMessageTrace)) {
this.enableMessageTrace = Boolean.parseBoolean(enableMessageTrace);
}
this.customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
this.accessChannel = properties.getProperty(RocketMQConstants.ROCKETMQ_ACCESS_CHANNEL);
this.namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
this.nameServer = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
String batchSize = properties.getProperty(RocketMQConstants.ROCKETMQ_BATCH_SIZE);
if (StringUtils.isNotEmpty(batchSize)) {
this.batchSize = Integer.parseInt(batchSize);
}
String subscribeFilter = properties.getProperty(RocketMQConstants.ROCKETMQ_SUBSCRIBE_FILTER);
if (StringUtils.isNotEmpty(subscribeFilter)) {
this.filter = subscribeFilter;
}
}
@Override
public void connect() {
RPCHook rpcHook = null;
if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(accessKey);
sessionCredentials.setSecretKey(secretKey);
rpcHook = new AclClientRPCHook(sessionCredentials);
}
rocketMQConsumer = new DefaultMQPushConsumer(groupName,
rpcHook,
new AllocateMessageQueueAveragely(),
enableMessageTrace,
customizedTraceTopic);
rocketMQConsumer.setVipChannelEnabled(false);
if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) {
rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(this.namespace)) {
rocketMQConsumer.setNamespace(this.namespace);
}
if (!StringUtils.isBlank(nameServer)) {
rocketMQConsumer.setNamesrvAddr(nameServer);
}
if (batchSize != -1) {
rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
}
try {
if (rocketMQConsumer == null) {
this.connect();
}
rocketMQConsumer.subscribe(this.topic, this.filter);
rocketMQConsumer.registerMessageListener((MessageListenerOrderly) (messageExts, context) -> {
context.setAutoCommit(true);
boolean isSuccess = process(messageExts);
if (isSuccess) {
return ConsumeOrderlyStatus.SUCCESS;
} else {
return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
}
});
rocketMQConsumer.start();
} catch (MQClientException ex) {
connected = false;
logger.error("Start RocketMQ consumer error", ex);
}
}
private boolean process(List<MessageExt> messageExts) {
if (logger.isDebugEnabled()) {
logger.debug("Get Message: {}", messageExts);
}
List<CommonMessage> messageList = Lists.newArrayList();
for (MessageExt messageExt : messageExts) {
byte[] data = messageExt.getBody();
if (data != null) {
try {
if (!flatMessage) {
Message message = CanalMessageSerializerUtil.deserializer(data);
messageList.addAll(MessageUtil.convert(message));
} else {
CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
messageList.add(commonMessage);
}
} catch (Exception ex) {
logger.error("Add message error", ex);
throw new CanalClientException(ex);
}
} else {
logger.warn("Received message data is null");
}
}
ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
try {
messageBlockingQueue.put(batchMessage);
} catch (InterruptedException e) {
logger.error("Put message to queue error", e);
throw new RuntimeException(e);
}
boolean isCompleted;
try {
isCompleted = batchMessage.waitFinish(batchProcessTimeout);
} catch (InterruptedException e) {
logger.error("Interrupted when waiting messages to be finished.", e);
throw new RuntimeException(e);
}
boolean isSuccess = batchMessage.isSuccess();
return isCompleted && isSuccess;
}
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
try {
if (this.lastGetBatchMessage != null) {
throw new CanalClientException("mq get/ack not support concurrent & async ack");
}
ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
if (batchMessage != null) {
this.lastGetBatchMessage = batchMessage;
return batchMessage.getData();
}
} catch (InterruptedException ex) {
logger.warn("Get message timeout", ex);
throw new CanalClientException("Failed to fetch the data after: " + timeout);
}
return null;
}
@Override
public void rollback() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void ack() {
try {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.ack();
}
} catch (Throwable e) {
if (this.lastGetBatchMessage != null) {
this.lastGetBatchMessage.fail();
}
} finally {
this.lastGetBatchMessage = null;
}
}
@Override
public void disconnect() {
rocketMQConsumer.unsubscribe(topic);
rocketMQConsumer.shutdown();
connected = false;
}
}

48
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java Normal file
View File

@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.connector.rocketmq.consumer;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
public class ConsumerBatchMessage<T> {
private final List<T> data;
private CountDownLatch latch;
private boolean hasFailure = false;
public ConsumerBatchMessage(List<T> data){
this.data = data;
latch = new CountDownLatch(1);
}
public boolean waitFinish(long timeout) throws InterruptedException {
return latch.await(timeout, TimeUnit.MILLISECONDS);
}
public boolean isSuccess() {
return !hasFailure;
}
public List<T> getData() {
return data;
}
/**
* Countdown if the sub message is successful.
*/
public void ack() {
latch.countDown();
}
/**
* Countdown and fail-fast if the sub message is failed.
*/
public void fail() {
hasFailure = true;
// fail fast
long count = latch.getCount();
for (int i = 0; i < count; i++) {
latch.countDown();
}
}
}

312
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java Normal file
View File

@@ -0,0 +1,312 @@
package com.alibaba.otter.canal.connector.rocketmq.producer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage;
/**
* RocketMQ Producer SPI 实现
*
* @author rewerma 2020-01-27
* @version 1.0.0
*/
@SPI("rocketmq")
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
@Override
public void init(Properties properties) {
RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
this.mqProperties = rocketMQProperties;
super.init(properties);
loadRocketMQProperties(properties);
RPCHook rpcHook = null;
if (mqProperties.getAliyunAccessKey().length() > 0
&& mqProperties.getAliyunSecretKey().length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
rpcHook = new AclClientRPCHook(sessionCredentials);
}
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
rpcHook,
rocketMQProperties.isEnableMessageTrace(),
rocketMQProperties.getCustomizedTraceTopic());
if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) {
defaultMQProducer.setNamespace(rocketMQProperties.getNamespace());
}
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed());
defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled());
logger.info("##Start RocketMQ producer##");
try {
defaultMQProducer.start();
} catch (MQClientException ex) {
throw new CanalException("Start RocketMQ producer error", ex);
}
}
private void loadRocketMQProperties(Properties properties) {
RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
if (!StringUtils.isEmpty(producerGroup)) {
rocketMQProperties.setProducerGroup(producerGroup);
}
String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
if (!StringUtils.isEmpty(enableMessageTrace)) {
rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
}
String customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
if (!StringUtils.isEmpty(customizedTraceTopic)) {
rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
}
String namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
if (!StringUtils.isEmpty(namespace)) {
rocketMQProperties.setNamespace(namespace);
}
String namesrvAddr = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
if (!StringUtils.isEmpty(namesrvAddr)) {
rocketMQProperties.setNamesrvAddr(namesrvAddr);
}
String retry = properties.getProperty(RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED);
if (!StringUtils.isEmpty(retry)) {
rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
}
String vipChannelEnabled = properties.getProperty(RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
if (!StringUtils.isEmpty(vipChannelEnabled)) {
rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
}
}
@Override
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(() -> {
try {
send(destination, topicName, messageSub);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
public void send(final MQDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) {
if (!mqProperties.isFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.isDatabaseHash());
int length = messages.length;
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < length; i++) {
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
if (dataPartition != null) {
final int index = i;
template.submit(() -> {
Message data = new Message(topicName,
CanalMessageSerializerUtil.serializer(dataPartition,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, index);
});
}
}
// 等所有分片发送完毕
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
Message data = new Message(topicName,
CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
sendMessage(data, partition);
}
} else {
// 并发构造
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
// 初始化分区合并队列
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
for (int i = 0; i < destination.getPartitionsNum(); i++) {
partitionFlatMessages.add(new ArrayList<>());
}
for (FlatMessage flatMessage : flatMessages) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.isDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
}
}
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < partitionFlatMessages.size(); i++) {
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
if (flatMessagePart != null) {
final int index = i;
template.submit(() -> {
List<Message> messages = flatMessagePart.stream()
.map(flatMessage -> new Message(topicName,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, index);
});
}
}
// 批量等所有分区的结果
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
List<Message> messages = flatMessages.stream()
.map(flatMessage -> new Message(topicName,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, partition);
}
}
}
private void sendMessage(Message message, int partition) {
try {
SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> {
if (partition > mqs.size()) {
return mqs.get(partition % mqs.size());
} else {
return mqs.get(partition);
}
}, null);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("deprecation")
private void sendMessage(List<Message> messages, int partition) {
if (messages.isEmpty()) {
return;
}
// 获取一下messageQueue
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
if (topicInfo == null) {
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
// 批量发送
List<MessageQueue> queues = topicInfo.getMessageQueueList();
int size = queues.size();
if (size <= 0) {
// 可能是第一次创建
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
MessageQueue queue;
if (partition > size) {
queue = queues.get(partition % size);
} else {
queue = queues.get(partition);
}
try {
SendResult sendResult = this.defaultMQProducer.send(messages, queue);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
}
}
@Override
public void stop() {
logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown();
super.stop();
}
}

1
connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer Normal file
View File

@@ -0,0 +1 @@
rocketmq=com.alibaba.otter.canal.connector.rocketmq.producer.CanalRocketMQProducer

1
connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer Normal file
View File

@@ -0,0 +1 @@
rocketmq=com.alibaba.otter.canal.connector.rocketmq.consumer.CanalRocketMQConsumer

124
connector/tcp-connector/pom.xml Normal file
View File

@@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.connector</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.tcp</artifactId>
<packaging>jar</packaging>
<name>canal connector tcp module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-aop</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.aliyun.openservices</groupId>
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin"
overwrite="true">
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
<include name="connector.tcp-${project.version}-jar-with-dependencies.jar"/>
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

12
connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java Normal file
View File

@@ -0,0 +1,12 @@
package com.alibaba.otter.canal.connector.tcp.config;
public class TCPConstants {
public static final String ROOT = "canal";
public static final String CANAL_TCP_HOST = ROOT + "." + "tcp.server.host";
public static final String CANAL_TCP_ZK_HOSTS = ROOT + "." + "tcp.zookeeper.hosts";
public static final String CANAL_TCP_USERNAME = ROOT + "." + "tcp.username";
public static final String CANAL_TCP_PASSWORD = ROOT + "." + "tcp.password";
public static final String CANAL_TCP_BATCH_SIZE = ROOT + "." + "tcp.batch.size";
}

106
connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java Normal file
View File

@@ -0,0 +1,106 @@
package com.alibaba.otter.canal.connector.tcp.consumer;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
import com.alibaba.otter.canal.protocol.Message;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
/**
* TCP 消费者连接器, 一个destination对应一个SPI实例
*
* @author rewerma 2020-01-30
* @version 1.0.0
*/
@SPI("tcp")
public class CanalTCPConsumer implements CanalMsgConsumer {
private static final String PREFIX_TCP_CONFIG = "canal.tcp.";
private Long currentBatchId = null;
private CanalConnector canalConnector;
private int batchSize = 500;
@Override
public void init(Properties properties, String destination, String groupId) {
// load config
String host = properties.getProperty(TCPConstants.CANAL_TCP_HOST);
String username = properties.getProperty(TCPConstants.CANAL_TCP_USERNAME);
String password = properties.getProperty(TCPConstants.CANAL_TCP_PASSWORD);
String zkHosts = properties.getProperty(TCPConstants.CANAL_TCP_ZK_HOSTS);
String batchSizePro = properties.getProperty(TCPConstants.CANAL_TCP_BATCH_SIZE);
if (batchSizePro != null) {
batchSize = Integer.parseInt(batchSizePro);
}
if (host != null) {
String[] ipPort = host.split(":");
SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
this.canalConnector = new SimpleCanalConnector(sa, username, password, destination);
} else {
this.canalConnector = new ClusterCanalConnector(username,
password,
destination,
new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkHosts)));
}
}
@Override
public void connect() {
canalConnector.connect();
canalConnector.subscribe();
}
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
try {
Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
long batchId = message.getId();
currentBatchId = batchId;
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
return null;
} else {
return MessageUtil.convert(message);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void rollback() {
if (currentBatchId != null && currentBatchId != -1) {
canalConnector.rollback(currentBatchId);
currentBatchId = null;
}
}
@Override
public void ack() {
if (currentBatchId != null) {
canalConnector.ack(currentBatchId);
currentBatchId = null;
}
}
@Override
public void disconnect() {
canalConnector.unsubscribe();
canalConnector.disconnect();
}
}

1
connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer Normal file
View File

@@ -0,0 +1 @@
tcp=com.alibaba.otter.canal.connector.tcp.consumer.CanalTCPConsumer

294
deployer/pom.xml
View File

@@ -1,127 +1,185 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<modelVersion>4.0.0</modelVersion> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent> <modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId> <parent>
<artifactId>canal</artifactId> <groupId>com.alibaba.otter</groupId>
<version>1.1.5-SNAPSHOT</version> <artifactId>canal</artifactId>
<relativePath>../pom.xml</relativePath> <version>1.1.5-SNAPSHOT</version>
</parent> <relativePath>../pom.xml</relativePath>
<groupId>com.alibaba.otter</groupId> </parent>
<artifactId>canal.deployer</artifactId> <groupId>com.alibaba.otter</groupId>
<packaging>jar</packaging> <artifactId>canal.deployer</artifactId>
<name>canal deployer module for otter ${project.version}</name> <packaging>jar</packaging>
<dependencies> <name>canal deployer module for otter ${project.version}</name>
<dependency> <dependencies>
<groupId>com.alibaba.otter</groupId> <dependency>
<artifactId>canal.server</artifactId> <groupId>com.alibaba.otter</groupId>
<version>${project.version}</version> <artifactId>canal.server</artifactId>
</dependency> <version>${project.version}</version>
</dependency>
<!-- 这里指定runtime的metrics provider--> <!-- 这里指定runtime的metrics provider-->
<dependency> <dependency>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal.prometheus</artifactId> <artifactId>canal.prometheus</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
</dependencies>
<build> <dependency>
<plugins> <groupId>com.alibaba.otter</groupId>
<!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 --> <artifactId>connector.kafka</artifactId>
<plugin> <version>${project.version}</version>
<artifactId>maven-jar-plugin</artifactId> <exclusions>
<configuration> <exclusion>
<archive> <artifactId>*</artifactId>
<addMavenDescriptor>true</addMavenDescriptor> <groupId>*</groupId>
</archive> </exclusion>
<excludes> </exclusions>
<exclude>**/logback.xml</exclude> <classifier>jar-with-dependencies</classifier>
<exclude>**/canal.properties</exclude> <scope>provided</scope>
<exclude>**/spring/**</exclude> </dependency>
<exclude>**/example/**</exclude> <dependency>
<exclude>**/mq.yml</exclude> <groupId>com.alibaba.otter</groupId>
</excludes> <artifactId>connector.rocketmq</artifactId>
</configuration> <version>${project.version}</version>
</plugin> <exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.rabbitmq</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
<classifier>jar-with-dependencies</classifier>
<scope>provided</scope>
</dependency>
</dependencies>
<plugin> <build>
<groupId>org.apache.maven.plugins</groupId> <plugins>
<artifactId>maven-assembly-plugin</artifactId> <plugin>
<!-- 这是最新版本，推荐使用这个版本 --> <artifactId>maven-jar-plugin</artifactId>
<version>2.2.1</version> <configuration>
<executions> <archive>
<execution> <addMavenDescriptor>true</addMavenDescriptor>
<id>assemble</id> </archive>
<goals> <excludes>
<goal>single</goal> <exclude>**/logback.xml</exclude>
</goals> <exclude>**/canal.properties</exclude>
<phase>package</phase> <exclude>**/spring/**</exclude>
</execution> <exclude>**/example/**</exclude>
</executions> <exclude>**/mq.yml</exclude>
<configuration> </excludes>
<appendAssemblyId>false</appendAssemblyId> </configuration>
<attach>false</attach> </plugin>
</configuration>
</plugin>
</plugins>
</build>
<profiles> <plugin>
<profile> <groupId>org.apache.maven.plugins</groupId>
<id>dev</id> <artifactId>maven-assembly-plugin</artifactId>
<activation> <!-- 这是最新版本，推荐使用这个版本 -->
<activeByDefault>true</activeByDefault> <version>2.2.1</version>
<property> <executions>
<name>env</name> <execution>
<value>!release</value> <id>assemble</id>
</property> <goals>
</activation> <goal>single</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<executions>
<execution>
<id>copy-dependencies-to-canal-client-service</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeClassifiers>jar-with-dependencies</includeClassifiers>
<outputDirectory>${project.basedir}/target/canal/plugin</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<build> <profiles>
<plugins> <profile>
<plugin> <id>dev</id>
<artifactId>maven-assembly-plugin</artifactId> <activation>
<configuration> <activeByDefault>true</activeByDefault>
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 --> <property>
<descriptors> <name>env</name>
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor> <value>!release</value>
</descriptors> </property>
<finalName>canal</finalName> </activation>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
</profile> <build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
<descriptors>
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
</descriptors>
<finalName>canal</finalName>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
<profile> </profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build> <profile>
<plugins> <id>release</id>
<plugin> <activation>
<artifactId>maven-assembly-plugin</artifactId> <property>
<configuration> <name>env</name>
<!-- 发布模式使用的maven assembly插件描述文件 --> <value>release</value>
<descriptors> </property>
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor> </activation>
</descriptors>
<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 --> <build>
<finalName>${project.artifactId}-${project.version}</finalName> <plugins>
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 --> <plugin>
<outputDirectory>${project.parent.build.directory}</outputDirectory> <artifactId>maven-assembly-plugin</artifactId>
</configuration> <configuration>
</plugin> <!-- 发布模式使用的maven assembly插件描述文件 -->
</plugins> <descriptors>
</build> <descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
</profile> </descriptors>
</profiles> <!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
<finalName>${project.artifactId}-${project.version}</finalName>
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
<outputDirectory>${project.parent.build.directory}</outputDirectory>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project> </project>

4
deployer/src/main/assembly/release.xml
View File

@@ -42,6 +42,10 @@
<exclude>**/*</exclude> <exclude>**/*</exclude>
</excludes> </excludes>
</fileSet> </fileSet>
<fileSet>
<directory>${project.basedir}/target/canal/plugin</directory>
<outputDirectory>/plugin/</outputDirectory>
</fileSet>
</fileSets> </fileSets>
<dependencySets> <dependencySets>
<dependencySet> <dependencySet>

57
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
View File

@@ -46,36 +46,37 @@ public class CanalConstants {
public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel"; public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";
public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace"; // public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel"; // public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic"; // public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace"; // public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable"; // public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath"; // public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath"; // public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username"; // public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password"; // public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost"; // public static final String CANAL_MQ_PARALLELTHREADSIZE = ROOT + "." + "mq.parallelThreadSize";
public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid"; // public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange"; // public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash"; // public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
// public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
// public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
// public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
// public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
// public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
// public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
// public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
// public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
// public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
// public static final String CANAL_MQ_USERNAME = ROOT + "." + "mq.username";
// public static final String CANAL_MQ_PASSWORD = ROOT + "." + "mq.password";
// public static final String CANAL_MQ_VHOST = ROOT + "." + "mq.vhost";
// public static final String CANAL_MQ_ALIYUN_UID = ROOT + "." + "mq.aliyunuid";
// public static final String CANAL_MQ_EXCHANGE = ROOT + "." + "mq.exchange";
// public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
public static String getInstanceModeKey(String destination) { public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

210
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java
View File

@@ -2,35 +2,36 @@ package com.alibaba.otter.canal.deployer;
import java.util.Properties; import java.util.Properties;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty; import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
import com.alibaba.otter.canal.common.MQProperties; import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import com.alibaba.otter.canal.deployer.admin.CanalAdminController; import com.alibaba.otter.canal.deployer.admin.CanalAdminController;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.rabbitmq.CanalRabbitMQProducer;
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
import com.alibaba.otter.canal.server.CanalMQStarter; import com.alibaba.otter.canal.server.CanalMQStarter;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/** /**
* Canal server 启动类 * Canal server 启动类
* *
* @author rewerma 2018-12-30 下午05:12:16 * @author rewerma 2020-01-27
* @version 1.0.1 * @version 1.0.2
*/ */
public class CanalStarter { public class CanalStarter {
private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class); private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
private CanalController controller = null; private static final String CONNECTOR_SPI_DIR = "/plugin";
private CanalMQProducer canalMQProducer = null; private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";
private Thread shutdownThread = null;
private CanalMQStarter canalMQStarter = null; private CanalController controller = null;
private CanalMQProducer canalMQProducer = null;
private Thread shutdownThread = null;
private CanalMQStarter canalMQStarter = null;
private volatile Properties properties; private volatile Properties properties;
private volatile boolean running = false; private volatile boolean running = false;
private CanalAdminWithNetty canalAdmin; private CanalAdminWithNetty canalAdmin;
@@ -61,20 +62,23 @@ public class CanalStarter {
*/ */
public synchronized void start() throws Throwable { public synchronized void start() throws Throwable {
String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE); String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
if (serverMode.equalsIgnoreCase("kafka")) { if (!"tcp".equalsIgnoreCase(serverMode)) {
canalMQProducer = new CanalKafkaProducer(); ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
} else if (serverMode.equalsIgnoreCase("rocketmq")) { canalMQProducer = loader
canalMQProducer = new CanalRocketMQProducer(); .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
} else if (serverMode.equalsIgnoreCase("rabbitmq")) { if (canalMQProducer != null) {
canalMQProducer = new CanalRabbitMQProducer(); ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
canalMQProducer.init(properties);
Thread.currentThread().setContextClassLoader(cl);
}
} }
MQProperties mqProperties = null;
if (canalMQProducer != null) { if (canalMQProducer != null) {
mqProperties = buildMQProperties(properties); MQProperties mqProperties = canalMQProducer.getMqProperties();
// disable netty // disable netty
System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true"); System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
if (mqProperties.getFlatMessage()) { if (mqProperties.isFlatMessage()) {
// 设置为raw避免ByteString->Entry的二次解析 // 设置为raw避免ByteString->Entry的二次解析
System.setProperty("canal.instance.memory.rawEntry", "false"); System.setProperty("canal.instance.memory.rawEntry", "false");
} }
@@ -104,7 +108,7 @@ public class CanalStarter {
if (canalMQProducer != null) { if (canalMQProducer != null) {
canalMQStarter = new CanalMQStarter(canalMQProducer); canalMQStarter = new CanalMQStarter(canalMQProducer);
String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS); String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
canalMQStarter.start(mqProperties, destinations); canalMQStarter.start(destinations);
controller.setCanalMQStarter(canalMQStarter); controller.setCanalMQStarter(canalMQStarter);
} }
@@ -119,11 +123,15 @@ public class CanalStarter {
String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP); String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP);
logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", port, user, passwd, ip); logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}",
port,
user,
passwd,
ip);
CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance(); CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
canalAdminWithNetty.setCanalAdmin(canalAdmin); canalAdminWithNetty.setCanalAdmin(canalAdmin);
canalAdminWithNetty.setPort(Integer.valueOf(port)); canalAdminWithNetty.setPort(Integer.parseInt(port));
canalAdminWithNetty.setIp(ip); canalAdminWithNetty.setIp(ip);
canalAdminWithNetty.start(); canalAdminWithNetty.start();
this.canalAdmin = canalAdminWithNetty; this.canalAdmin = canalAdminWithNetty;
@@ -162,156 +170,4 @@ public class CanalStarter {
} }
running = false; running = false;
} }
/**
* 构造MQ对应的配置
*
* @param properties canal.properties 配置
* @return
*/
private static MQProperties buildMQProperties(Properties properties) {
MQProperties mqProperties = new MQProperties();
String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
if (!StringUtils.isEmpty(servers)) {
mqProperties.setServers(servers);
}
String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
if (!StringUtils.isEmpty(retires)) {
mqProperties.setRetries(Integer.valueOf(retires));
}
String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
if (!StringUtils.isEmpty(batchSize)) {
mqProperties.setBatchSize(Integer.valueOf(batchSize));
}
String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
if (!StringUtils.isEmpty(lingerMs)) {
mqProperties.setLingerMs(Integer.valueOf(lingerMs));
}
String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
if (!StringUtils.isEmpty(maxRequestSize)) {
mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
}
String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
if (!StringUtils.isEmpty(bufferMemory)) {
mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
}
String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
if (!StringUtils.isEmpty(canalBatchSize)) {
mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
}
String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
if (!StringUtils.isEmpty(canalGetTimeout)) {
mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
}
String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
}
String parallelThreadSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PARALLELTHREADSIZE);
if (!StringUtils.isEmpty(parallelThreadSize)) {
mqProperties.setParallelThreadSize(Integer.valueOf(parallelThreadSize));
}
String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
if (!StringUtils.isEmpty(compressionType)) {
mqProperties.setCompressionType(compressionType);
}
String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks);
}
String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunSecretKey(aliyunSecretKey);
}
String producerGroup = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PRODUCERGROUP);
if (!StringUtils.isEmpty(producerGroup)) {
mqProperties.setProducerGroup(producerGroup);
}
String enableMessageTrace = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE);
if (!StringUtils.isEmpty(enableMessageTrace)) {
mqProperties.setEnableMessageTrace(Boolean.valueOf(enableMessageTrace));
}
String accessChannel = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
if (!StringUtils.isEmpty(accessChannel)) {
mqProperties.setAccessChannel(accessChannel);
}
String customizedTraceTopic = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC);
if (!StringUtils.isEmpty(customizedTraceTopic)) {
mqProperties.setCustomizedTraceTopic(customizedTraceTopic);
}
String namespace = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_NAMESPACE);
if (!StringUtils.isEmpty(namespace)) {
mqProperties.setNamespace(namespace);
}
String kafkaKerberosEnable = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
if (!StringUtils.isEmpty(kafkaKerberosEnable)) {
mqProperties.setKerberosEnable(Boolean.valueOf(kafkaKerberosEnable));
}
String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH);
if (!StringUtils.isEmpty(kafkaKerberosKrb5Filepath)) {
mqProperties.setKerberosKrb5FilePath(kafkaKerberosKrb5Filepath);
}
String kafkaKerberosJaasFilepath = CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH);
if (!StringUtils.isEmpty(kafkaKerberosJaasFilepath)) {
mqProperties.setKerberosJaasFilePath(kafkaKerberosJaasFilepath);
}
String vhost = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_VHOST);
if (!StringUtils.isEmpty(vhost)) {
mqProperties.setVhost(vhost);
}
String username = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_USERNAME);
if (!StringUtils.isEmpty(username)) {
mqProperties.setUsername(username);
}
String password = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PASSWORD);
if (!StringUtils.isEmpty(password)) {
mqProperties.setPassword(password);
}
String aliyunUID = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ALIYUN_UID);
if (!StringUtils.isEmpty(aliyunUID)) {
mqProperties.setAliyunUID(Long.valueOf(aliyunUID));
}
String exchange = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_EXCHANGE);
if (!StringUtils.isEmpty(exchange)) {
mqProperties.setExchange(exchange);
}
String databaseHash = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_DATABASE_HASH);
if (!StringUtils.isEmpty(databaseHash)){
mqProperties.setDatabaseHash(Boolean.valueOf(databaseHash));
}
for (Object key : properties.keySet()) {
key = StringUtils.trim(key.toString());
if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) {
String value = CanalController.getProperty(properties, (String) key);
String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1);
mqProperties.getProperties().put(subKey, value);
}
}
return mqProperties;
}
} }

79
deployer/src/main/resources/canal.properties
View File

@@ -21,7 +21,7 @@ canal.zkServers =
# flush data to zk # flush data to zk
canal.zookeeper.flush.period = 1000 canal.zookeeper.flush.period = 1000
canal.withoutNetty = false canal.withoutNetty = false
# tcp, kafka, RocketMQ # tcp, kafka, rocketMQ, rabbitMQ
canal.serverMode = tcp canal.serverMode = tcp
# flush meta cursor/parse position to file # flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir} canal.file.data.dir = ${canal.conf.dir}
@@ -86,10 +86,6 @@ canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days) # purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360 canal.instance.tsdb.snapshot.expire = 360
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
################################################# #################################################
######### destinations ############# ######### destinations #############
################################################# #################################################
@@ -111,35 +107,54 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml #canal.instance.global.spring.xml = classpath:spring/default-instance.xml
################################################## ##################################################
######### MQ ############# ######### MQ Properties #############
################################################## ##################################################
canal.mq.servers = 127.0.0.1:6667 canal.mq.flat.message = true
canal.mq.retries = 0
canal.mq.batchSize = 16384
canal.mq.maxRequestSize = 1048576
canal.mq.lingerMs = 100
canal.mq.bufferMemory = 33554432
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
canal.mq.parallelThreadSize = 8
canal.mq.flatMessage = true
canal.mq.compressionType = none
canal.mq.acks = all
#canal.mq.properties. =
canal.mq.producerGroup = test
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local
# aliyun mq namespace
#canal.mq.namespace =
canal.mq.vhost=
canal.mq.exchange=
canal.mq.username=
canal.mq.password=
canal.mq.aliyunuid=
canal.mq.database.hash = true canal.mq.database.hash = true
canal.mq.parallel.thread.size = 8
canal.mq.canal.batch.size = 50
canal.mq.canal.fetch.timeout = 100
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.access.channel = local
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
canal.aliyun.uid=
################################################## ##################################################
######### Kafka Kerberos Info ############# ######### Kafka #############
################################################## ##################################################
kafka.bootstrap.servers = 127.0.0.1:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0
canal.mq.kafka.kerberos.enable = false canal.mq.kafka.kerberos.enable = false
canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf" canal.mq.kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf" canal.mq.kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
##################################################
######### RocketMQ #############
##################################################
rocketmq.producer.group = test
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic =
rocketmq.namespace =
rocketmq.namesrv.addr = 127.0.0.1:9876
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
##################################################
######### RabbitMQ #############
##################################################
rabbitmq.host =
rabbitmq.virtual.host =
rabbitmq.exchange =
rabbitmq.username =
rabbitmq.password =

8
deployer/src/main/resources/logback.xml
View File

@@ -73,11 +73,15 @@
<level value="INFO" /> <level value="INFO" />
<appender-ref ref="CANAL-META" /> <appender-ref ref="CANAL-META" />
</logger> </logger>
<logger name="com.alibaba.otter.canal.kafka" additivity="false"> <logger name="com.alibaba.otter.canal.connector.kafka" additivity="false">
<level value="INFO" /> <level value="INFO" />
<appender-ref ref="CANAL-ROOT" /> <appender-ref ref="CANAL-ROOT" />
</logger> </logger>
<logger name="com.alibaba.otter.canal.rabbitmq" additivity="false"> <logger name="com.alibaba.otter.canal.connector.rocketmq" additivity="false">
<level value="INFO" />
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.connector.rabbitmq" additivity="false">
<level value="INFO" /> <level value="INFO" />
<appender-ref ref="CANAL-ROOT" /> <appender-ref ref="CANAL-ROOT" />
</logger> </logger>

28
pom.xml
View File

@@ -100,8 +100,6 @@
<file_encoding>UTF-8</file_encoding> <file_encoding>UTF-8</file_encoding>
<javadoc_skip>true</javadoc_skip> <javadoc_skip>true</javadoc_skip>
<spring_version>3.2.18.RELEASE</spring_version> <spring_version>3.2.18.RELEASE</spring_version>
<rocketmq_version>4.5.2</rocketmq_version>
<rabbitmq_version>5.5.0</rabbitmq_version>
<maven-jacoco-plugin.version>0.8.3</maven-jacoco-plugin.version> <maven-jacoco-plugin.version>0.8.3</maven-jacoco-plugin.version>
<maven-surefire.version>2.22.1</maven-surefire.version> <maven-surefire.version>2.22.1</maven-surefire.version>
<argline>-server -Xms512m -Xmx1024m -Dfile.encoding=UTF-8 <argline>-server -Xms512m -Xmx1024m -Dfile.encoding=UTF-8
@@ -129,6 +127,7 @@
<module>prometheus</module> <module>prometheus</module>
<module>admin</module> <module>admin</module>
<module>client-adapter</module> <module>client-adapter</module>
<module>connector</module>
</modules> </modules>
<dependencyManagement> <dependencyManagement>
@@ -233,6 +232,11 @@
<artifactId>aviator</artifactId> <artifactId>aviator</artifactId>
<version>2.2.1</version> <version>2.2.1</version>
</dependency> </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.58.sec06</version>
</dependency>
<dependency> <dependency>
<groupId>oro</groupId> <groupId>oro</groupId>
<artifactId>oro</artifactId> <artifactId>oro</artifactId>
@@ -336,26 +340,6 @@
<artifactId>jsr305</artifactId> <artifactId>jsr305</artifactId>
<version>3.0.2</version> <version>3.0.2</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
<version>${rocketmq_version}</version>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>${rabbitmq_version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
<version>1.0.3</version>
</dependency>
<dependency> <dependency>
<groupId>javax.annotation</groupId> <groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId> <artifactId>javax.annotation-api</artifactId>

36
server/pom.xml
View File

@@ -25,39 +25,15 @@
<artifactId>canal.instance.manager</artifactId> <artifactId>canal.instance.manager</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<!-- kafka -->
<dependency> <dependency>
<groupId>org.apache.kafka</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>kafka_2.11</artifactId> <artifactId>connector.core</artifactId>
<version>1.1.1</version> <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.commons</groupId>
<artifactId>rocketmq-client</artifactId> <artifactId>commons-lang3</artifactId>
</dependency> <version>3.7</version>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
</dependency>
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
<dependency>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
<version>3.2.2.Final</version>
</dependency> </dependency>
<!-- test dependency --> <!-- test dependency -->

37
server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java
View File

@@ -1,37 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.spi.CanalMQProducer;
/**
* @author agapple 2019年9月29日 上午11:17:11
* @since 1.1.5
*/
public abstract class AbstractMQProducer implements CanalMQProducer {
protected ThreadPoolExecutor executor;
@Override
public void init(MQProperties mqProperties) {
int parallelThreadSize = mqProperties.getParallelThreadSize();
executor = new ThreadPoolExecutor(parallelThreadSize,
parallelThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
new NamedThreadFactory("MQParallel"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@Override
public void stop() {
executor.shutdownNow();
executor = null;
}
}

368
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -1,368 +0,0 @@
package com.alibaba.otter.canal.common;
import java.util.Properties;
/**
* MQ 配置项
*
* @author machengyuan 2018-6-11 下午05:30:49
* @version 1.0.0
*/
public class MQProperties {
private String servers = "127.0.0.1:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private int maxRequestSize = 1048576;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout = 100L;
private boolean flatMessage = true;
private String compressionType = "none";
private String acks = "all";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private Properties properties = new Properties();
private boolean enableMessageTrace = false;
private String accessChannel = null;
private String customizedTraceTopic = null;
private String namespace = "";
// kafka集群是否启动Kerberos认证
private boolean kerberosEnable = false;
// 启动Kerberos认证时配置为krb5.conf文件的路径
private String kerberosKrb5FilePath = "";
// 启动Kerberos认证时配置为jaas.conf文件的路径
private String kerberosJaasFilePath = "";
// rabbitmq 账号
private String username = "";
// rabbitmq 密码
private String password = "";
// rabbitmq 密码
private String vhost = "";
// aliyun 用户ID rabbitmq 阿里云需要使用
private long aliyunUID = 0;
// rabbitmq 交换机
private String exchange = "";
// 消息发送的并行度
private int parallelThreadSize = 8;
// 是否取消根据database进行hash
private boolean databaseHash = true;
public static class CanalDestination {
private String canalDestination;
private String topic;
private Integer partition;
private Integer partitionsNum;
private String partitionHash;
private String dynamicTopic;
public String getCanalDestination() {
return canalDestination;
}
public void setCanalDestination(String canalDestination) {
this.canalDestination = canalDestination;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Integer getPartition() {
return partition;
}
public void setPartition(Integer partition) {
this.partition = partition;
}
public Integer getPartitionsNum() {
return partitionsNum;
}
public void setPartitionsNum(Integer partitionsNum) {
this.partitionsNum = partitionsNum;
}
public String getPartitionHash() {
return partitionHash;
}
public void setPartitionHash(String partitionHash) {
this.partitionHash = partitionHash;
}
public String getDynamicTopic() {
return dynamicTopic;
}
public void setDynamicTopic(String dynamicTopic) {
this.dynamicTopic = dynamicTopic;
}
}
public String getServers() {
return servers;
}
public void setServers(String servers) {
this.servers = servers;
}
public int getRetries() {
return retries;
}
public void setRetries(int retries) {
this.retries = retries;
}
public int getBatchSize() {
return batchSize;
}
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
public int getLingerMs() {
return lingerMs;
}
public void setLingerMs(int lingerMs) {
this.lingerMs = lingerMs;
}
public long getBufferMemory() {
return bufferMemory;
}
public void setBufferMemory(long bufferMemory) {
this.bufferMemory = bufferMemory;
}
public int getCanalBatchSize() {
return canalBatchSize;
}
public void setCanalBatchSize(int canalBatchSize) {
this.canalBatchSize = canalBatchSize;
}
public Long getCanalGetTimeout() {
return canalGetTimeout;
}
public void setCanalGetTimeout(Long canalGetTimeout) {
this.canalGetTimeout = canalGetTimeout;
}
public boolean getFlatMessage() {
return flatMessage;
}
public void setFlatMessage(boolean flatMessage) {
this.flatMessage = flatMessage;
}
public boolean isFilterTransactionEntry() {
return filterTransactionEntry;
}
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
this.filterTransactionEntry = filterTransactionEntry;
}
public String getProducerGroup() {
return producerGroup;
}
public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup;
}
public String getCompressionType() {
return compressionType;
}
public void setCompressionType(String compressionType) {
this.compressionType = compressionType;
}
public String getAcks() {
return acks;
}
public void setAcks(String acks) {
this.acks = acks;
}
public String getAliyunAccessKey() {
return aliyunAccessKey;
}
public void setAliyunAccessKey(String aliyunAccessKey) {
this.aliyunAccessKey = aliyunAccessKey;
}
public String getAliyunSecretKey() {
return aliyunSecretKey;
}
public void setAliyunSecretKey(String aliyunSecretKey) {
this.aliyunSecretKey = aliyunSecretKey;
}
public int getMaxRequestSize() {
return maxRequestSize;
}
public void setMaxRequestSize(int maxRequestSize) {
this.maxRequestSize = maxRequestSize;
}
public Properties getProperties() {
return properties;
}
public void setProperties(Properties properties) {
this.properties = properties;
}
public boolean isEnableMessageTrace() {
return enableMessageTrace;
}
public void setEnableMessageTrace(boolean enableMessageTrace) {
this.enableMessageTrace = enableMessageTrace;
}
public String getAccessChannel() {
return accessChannel;
}
public void setAccessChannel(String accessChannel) {
this.accessChannel = accessChannel;
}
public String getCustomizedTraceTopic() {
return customizedTraceTopic;
}
public void setCustomizedTraceTopic(String customizedTraceTopic) {
this.customizedTraceTopic = customizedTraceTopic;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
public boolean isKerberosEnable() {
return kerberosEnable;
}
public void setKerberosEnable(boolean kerberosEnable) {
this.kerberosEnable = kerberosEnable;
}
public String getKerberosKrb5FilePath() {
return kerberosKrb5FilePath;
}
public void setKerberosKrb5FilePath(String kerberosKrb5FilePath) {
this.kerberosKrb5FilePath = kerberosKrb5FilePath;
}
public String getKerberosJaasFilePath() {
return kerberosJaasFilePath;
}
public void setKerberosJaasFilePath(String kerberosJaasFilePath) {
this.kerberosJaasFilePath = kerberosJaasFilePath;
}
public void setPassword(String password) {
this.password = password;
}
public String getPassword() {
return password;
}
public void setUsername(String username) {
this.username = username;
}
public String getUsername() {
return username;
}
public void setVhost(String vhost) {
this.vhost = vhost;
}
public String getVhost() {
return vhost;
}
public long getAliyunUID() {
return aliyunUID;
}
public void setAliyunUID(long aliyunUID) {
this.aliyunUID = aliyunUID;
}
public String getExchange() {
return exchange;
}
public void setExchange(String exchange) {
this.exchange = exchange;
}
public int getParallelThreadSize() {
return parallelThreadSize;
}
public void setParallelThreadSize(int parallelThreadSize) {
this.parallelThreadSize = parallelThreadSize;
}
public boolean getDatabaseHash() {
return databaseHash;
}
public void setDatabaseHash(boolean databaseHash) {
this.databaseHash = databaseHash;
}
@Override
public String toString() {
return "MQProperties [servers=" + servers + ", retries=" + retries + ", batchSize=" + batchSize + ", lingerMs="
+ lingerMs + ", maxRequestSize=" + maxRequestSize + ", bufferMemory=" + bufferMemory
+ ", filterTransactionEntry=" + filterTransactionEntry + ", producerGroup=" + producerGroup
+ ", canalBatchSize=" + canalBatchSize + ", canalGetTimeout=" + canalGetTimeout + ", flatMessage="
+ flatMessage + ", compressionType=" + compressionType + ", acks=" + acks + ", aliyunAccessKey="
+ aliyunAccessKey + ", aliyunSecretKey=" + aliyunSecretKey + ", properties=" + properties
+ ", enableMessageTrace=" + enableMessageTrace + ", accessChannel=" + accessChannel
+ ", customizedTraceTopic=" + customizedTraceTopic + ", namespace=" + namespace + ", kerberosEnable="
+ kerberosEnable + ", kerberosKrb5FilePath=" + kerberosKrb5FilePath + ", kerberosJaasFilePath="
+ kerberosJaasFilePath + ", username=" + username + ", password=" + password + ", vhost=" + vhost
+ ", aliyunUID=" + aliyunUID + ", exchange=" + exchange + ", parallelThreadSize=" + parallelThreadSize
+ ",databaseHash=" + databaseHash + "]";
}
}

145
server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java
View File

@@ -1,145 +0,0 @@
package com.alibaba.otter.canal.rabbitmq;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
private MQProperties mqProperties;
private Connection connect;
private Channel channel;
@Override
public void init(MQProperties mqProperties) {
super.init(mqProperties);
this.mqProperties = mqProperties;
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(mqProperties.getServers());
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
&& mqProperties.getAliyunUID() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
mqProperties.getAliyunSecretKey(),
mqProperties.getAliyunUID()));
} else {
factory.setUsername(mqProperties.getUsername());
factory.setPassword(mqProperties.getPassword());
}
factory.setVirtualHost(mqProperties.getVhost());
try {
connect = factory.newConnection();
channel = connect.createChannel();
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Start RabbitMQ producer error", ex);
}
}
@Override
public void send(final MQProperties.CanalDestination canalDestination, Message message, Callback callback)
throws IOException {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
canalDestination.getTopic(),
canalDestination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
send(canalDestination, topicName, messageSub);
}
});
}
template.waitForResult();
} else {
send(canalDestination, canalDestination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
private void send(MQProperties.CanalDestination canalDestination, String topicName, Message messageSub) {
if (!mqProperties.getFlatMessage()) {
byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry());
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
} else {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}",
message,
canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
}
}
}
}
private void sendMessage(String queueName, byte[] message) {
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
try {
channel.basicPublish(mqProperties.getExchange(), queueName, null, message);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void stop() {
logger.info("## Stop RabbitMQ producer##");
try {
this.connect.close();
this.channel.close();
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Stop RabbitMQ producer error", ex);
}
super.stop();
}
}

285
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
View File

@@ -1,285 +0,0 @@
package com.alibaba.otter.canal.rocketmq;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractMQProducer;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
private DefaultMQProducer defaultMQProducer;
private MQProperties mqProperties;
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
@Override
public void init(MQProperties rocketMQProperties) {
super.init(rocketMQProperties);
this.mqProperties = rocketMQProperties;
RPCHook rpcHook = null;
if (rocketMQProperties.getAliyunAccessKey().length() > 0
&& rocketMQProperties.getAliyunSecretKey().length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
rpcHook = new AclClientRPCHook(sessionCredentials);
}
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
rpcHook,
mqProperties.isEnableMessageTrace(),
mqProperties.getCustomizedTraceTopic());
if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(mqProperties.getNamespace())) {
defaultMQProducer.setNamespace(mqProperties.getNamespace());
}
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
defaultMQProducer.setVipChannelEnabled(false);
logger.info("##Start RocketMQ producer##");
try {
defaultMQProducer.start();
} catch (MQClientException ex) {
throw new CanalServerException("Start RocketMQ producer error", ex);
}
}
@Override
public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(executor);
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(data,
destination.getTopic(),
destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
try {
send(destination, topicName, messageSub);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
template.waitForResult();
} else {
send(destination, destination.getTopic(), data);
}
callback.commit();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
callback.rollback();
} finally {
template.clear();
}
}
public void send(final MQProperties.CanalDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) throws Exception {
if (!mqProperties.getFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
message.getId(),
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.getDatabaseHash());
int length = messages.length;
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < length; i++) {
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
if (dataPartition != null) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
Message data = new Message(topicName, CanalMessageSerializer.serializer(dataPartition,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, index);
}
});
}
}
// 等所有分片发送完毕
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
Message data = new Message(topicName, CanalMessageSerializer.serializer(message,
mqProperties.isFilterTransactionEntry()));
sendMessage(data, partition);
}
} else {
// 并发构造
EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
// 串行分区
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
if (flatMessages != null) {
// 初始化分区合并队列
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
List<List<FlatMessage>> partitionFlatMessages = new ArrayList<List<FlatMessage>>();
for (int i = 0; i < destination.getPartitionsNum(); i++) {
partitionFlatMessages.add(new ArrayList<FlatMessage>());
}
for (FlatMessage flatMessage : flatMessages) {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
destination.getPartitionsNum(),
destination.getPartitionHash(),
mqProperties.getDatabaseHash());
int length = partitionFlatMessage.length;
for (int i = 0; i < length; i++) {
partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
}
}
ExecutorTemplate template = new ExecutorTemplate(executor);
for (int i = 0; i < partitionFlatMessages.size(); i++) {
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
if (flatMessagePart != null) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
List<Message> messages = flatMessagePart.stream()
.map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, index);
}
});
}
}
// 批量等所有分区的结果
template.waitForResult();
} else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
List<Message> messages = flatMessages.stream()
.map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList());
// 批量发送
sendMessage(messages, partition);
}
}
}
}
private void sendMessage(Message message, int partition) {
try {
SendResult sendResult = this.defaultMQProducer.send(message, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
if (partition > mqs.size()) {
return mqs.get(partition % mqs.size());
} else {
return mqs.get(partition);
}
}
}, null);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("deprecation")
private void sendMessage(List<Message> messages, int partition) {
if (messages.isEmpty()) {
return;
}
// 获取一下messageQueue
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
if (topicInfo == null) {
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
// 批量发送
List<MessageQueue> queues = topicInfo.getMessageQueueList();
int size = queues.size();
if (size <= 0) {
// 可能是第一次创建
for (Message message : messages) {
sendMessage(message, partition);
}
} else {
MessageQueue queue = null;
if (partition > size) {
queue = queues.get(partition % size);
} else {
queue = queues.get(partition);
}
try {
SendResult sendResult = this.defaultMQProducer.send(messages, queue);
if (logger.isDebugEnabled()) {
logger.debug("Send Message Result: {}", sendResult);
}
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
}
}
@Override
public void stop() {
logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown();
super.stop();
}
}

30
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -7,18 +7,21 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.slf4j.MDC; import org.slf4j.MDC;
import com.alibaba.otter.canal.common.MQProperties; import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalMQConfig; import com.alibaba.otter.canal.instance.core.CanalMQConfig;
import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
public class CanalMQStarter { public class CanalMQStarter {
@@ -30,7 +33,7 @@ public class CanalMQStarter {
private CanalMQProducer canalMQProducer; private CanalMQProducer canalMQProducer;
private MQProperties properties; private MQProperties mqProperties;
private CanalServerWithEmbedded canalServer; private CanalServerWithEmbedded canalServer;
@@ -42,15 +45,14 @@ public class CanalMQStarter {
this.canalMQProducer = canalMQProducer; this.canalMQProducer = canalMQProducer;
} }
public synchronized void start(MQProperties properties, String destinations) { public synchronized void start(String destinations) {
try { try {
if (running) { if (running) {
return; return;
} }
this.properties = properties; mqProperties = canalMQProducer.getMqProperties();
canalMQProducer.init(properties);
// set filterTransactionEntry // set filterTransactionEntry
if (properties.isFilterTransactionEntry()) { if (mqProperties.isFilterTransactionEntry()) {
System.setProperty("canal.instance.filter.transaction.entry", "true"); System.setProperty("canal.instance.filter.transaction.entry", "true");
} }
@@ -151,7 +153,7 @@ public class CanalMQStarter {
} }
continue; continue;
} }
MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination(); MQDestination canalDestination = new MQDestination();
canalDestination.setCanalDestination(destination); canalDestination.setCanalDestination(destination);
CanalMQConfig mqConfig = canalInstance.getMqConfig(); CanalMQConfig mqConfig = canalInstance.getMqConfig();
canalDestination.setTopic(mqConfig.getTopic()); canalDestination.setTopic(mqConfig.getTopic());
@@ -163,15 +165,13 @@ public class CanalMQStarter {
canalServer.subscribe(clientIdentity); canalServer.subscribe(clientIdentity);
logger.info("## the MQ producer: {} is running now ......", destination); logger.info("## the MQ producer: {} is running now ......", destination);
Long getTimeout = properties.getCanalGetTimeout(); Integer getTimeout = mqProperties.getFetchTimeout();
int getBatchSize = properties.getCanalBatchSize(); Integer getBatchSize = mqProperties.getBatchSize();
while (running && destinationRunning.get()) { while (running && destinationRunning.get()) {
Message message; Message message;
if (getTimeout != null && getTimeout > 0) { if (getTimeout != null && getTimeout > 0) {
message = canalServer.getWithoutAck(clientIdentity, message = canalServer
getBatchSize, .getWithoutAck(clientIdentity, getBatchSize, getTimeout.longValue(), TimeUnit.MILLISECONDS);
getTimeout,
TimeUnit.MILLISECONDS);
} else { } else {
message = canalServer.getWithoutAck(clientIdentity, getBatchSize); message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
} }
@@ -180,7 +180,7 @@ public class CanalMQStarter {
try { try {
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) { if (batchId != -1 && size != 0) {
canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() { canalMQProducer.send(canalDestination, message, new Callback() {
@Override @Override
public void commit() { public void commit() {

37
server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java
View File

@@ -1,37 +0,0 @@
package com.alibaba.otter.canal.spi;
import java.io.IOException;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.Message;
public interface CanalMQProducer {
/**
* Init producer.
*
* @param mqProperties MQ config
*/
void init(MQProperties mqProperties);
/**
* Send canal message to related topic
*
* @param canalDestination canal mq destination
* @param message canal message
* @throws IOException
*/
void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException;
/**
* Stop MQ producer service
*/
void stop();
interface Callback {
void commit();
void rollback();
}
}