chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

模块化mq connector (#2562)

Browse Source 
* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
2020-02-28 11:00:08 +08:00
committed by GitHub
parent 4fc7e5f7d9
commit 265a8e8594
82 changed files with 4932 additions and 2412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
client-adapter/common/pom.xml
View File

@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId>
@@ -17,6 +18,11 @@
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>connector.core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
@@ -49,7 +55,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
</dependency>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>

13
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* 配置信息类
@@ -27,7 +28,7 @@ public class CanalClientConfig {
// 重试次数
private Integer retries;
// 消费超时时间
private Long timeout;
private Long timeout = 500L;
// 模式 tcp kafka rocketMQ
private String mode = "tcp";
// aliyun ak/sk
@@ -39,6 +40,8 @@ public class CanalClientConfig {
// rabbitmq vhost
private String vhost = "/";
private Properties consumerProperties;
private Long resourceOwnerId;
// 是否启用消息轨迹
private boolean enableMessageTrace;
@@ -163,6 +166,14 @@ public class CanalClientConfig {
this.vhost = vhost;
}
public Properties getConsumerProperties() {
return consumerProperties;
}
public void setConsumerProperties(Properties consumerProperties) {
this.consumerProperties = consumerProperties;
}
public Long getResourceOwnerId() {
return resourceOwnerId;
}

40
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
@@ -131,10 +132,13 @@ public class MessageUtil {
return dmls;
}
public static List<Dml> flatMessage2Dml(String destination, String groupId, List<FlatMessage> flatMessages) {
List<Dml> dmls = new ArrayList<Dml>(flatMessages.size());
for (FlatMessage flatMessage : flatMessages) {
Dml dml = flatMessage2Dml(destination, groupId, flatMessage);
public static List<Dml> flatMessage2Dml(String destination, String groupId, List<CommonMessage> commonMessages) {
if (commonMessages == null) {
return new ArrayList<>();
}
List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
for (CommonMessage commonMessage : commonMessages) {
Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
if (dml != null) {
dmls.add(dml);
}
@@ -143,31 +147,31 @@ public class MessageUtil {
return dmls;
}
public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) {
if (flatMessage == null) {
public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) {
if (commonMessage == null) {
return null;
}
Dml dml = new Dml();
dml.setDestination(destination);
dml.setGroupId(groupId);
dml.setDatabase(flatMessage.getDatabase());
dml.setTable(flatMessage.getTable());
dml.setPkNames(flatMessage.getPkNames());
dml.setIsDdl(flatMessage.getIsDdl());
dml.setType(flatMessage.getType());
dml.setTs(flatMessage.getTs());
dml.setEs(flatMessage.getEs());
dml.setSql(flatMessage.getSql());
dml.setDatabase(commonMessage.getDatabase());
dml.setTable(commonMessage.getTable());
dml.setPkNames(commonMessage.getPkNames());
dml.setIsDdl(commonMessage.getIsDdl());
dml.setType(commonMessage.getType());
dml.setTs(commonMessage.getTs());
dml.setEs(commonMessage.getEs());
dml.setSql(commonMessage.getSql());
// if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
// throw new RuntimeException("SqlType or mysqlType is null");
// }
List<Map<String, String>> data = flatMessage.getData();
List<Map<String, Object>> data = commonMessage.getData();
if (data != null) {
dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
dml.setData(data);
}
List<Map<String, String>> old = flatMessage.getOld();
List<Map<String, Object>> old = commonMessage.getOld();
if (old != null) {
dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
dml.setOld(old);
}
return dml;
}