chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
agapple
2020-02-28 11:40:14 +08:00
parent ab74b2c82c
commit 523e8b85e6
43 changed files with 298 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/EbeanConfig.java
View File

@@ -1,5 +1,10 @@
package com.alibaba.otter.canal.admin.config; package com.alibaba.otter.canal.admin.config;
import io.ebean.EbeanServer;
import io.ebean.EbeanServerFactory;
import io.ebean.config.ServerConfig;
import io.ebean.config.UnderscoreNamingConvention;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -8,11 +13,6 @@ import javax.sql.DataSource;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import io.ebean.EbeanServer;
import io.ebean.EbeanServerFactory;
import io.ebean.config.ServerConfig;
import io.ebean.config.UnderscoreNamingConvention;
/** /**
* Ebean 配置 * Ebean 配置
* *

5
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java
View File

@@ -2,11 +2,12 @@ package com.alibaba.otter.canal.admin.connector;
import java.util.function.Function; import java.util.function.Function;
import com.alibaba.otter.canal.admin.config.SpringContext;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment; import org.springframework.core.env.Environment;
import com.alibaba.otter.canal.admin.config.SpringContext;
public class SimpleAdminConnectors { public class SimpleAdminConnectors {
private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnectors.class); private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnectors.class);
@@ -25,7 +26,7 @@ public class SimpleAdminConnectors {
connector.connect(); connector.connect();
return function.apply(connector); return function.apply(connector);
} catch (Exception e) { } catch (Exception e) {
logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user,passwd); logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user, passwd);
logger.error(e.getMessage()); logger.error(e.getMessage());
} finally { } finally {
connector.disconnect(); connector.disconnect();

11
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalClusterController.java
View File

@@ -4,7 +4,14 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
@@ -22,7 +29,7 @@ public class CanalClusterController {
CanalClusterService canalClusterServic; CanalClusterService canalClusterServic;
@Autowired @Autowired
NodeServerService nodeServerService; NodeServerService nodeServerService;
@GetMapping(value = "/clusters") @GetMapping(value = "/clusters")
public BaseModel<List<CanalCluster>> clusters(CanalCluster canalCluster, @PathVariable String env) { public BaseModel<List<CanalCluster>> clusters(CanalCluster canalCluster, @PathVariable String env) {

2
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalConfigController.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.admin.controller; package com.alibaba.otter.canal.admin.controller;
import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
@@ -9,6 +8,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
import com.alibaba.otter.canal.admin.model.BaseModel; import com.alibaba.otter.canal.admin.model.BaseModel;
import com.alibaba.otter.canal.admin.model.CanalConfig; import com.alibaba.otter.canal.admin.model.CanalConfig;
import com.alibaba.otter.canal.admin.service.CanalConfigService; import com.alibaba.otter.canal.admin.service.CanalConfigService;

5
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java
View File

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.admin.controller;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -15,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
import com.alibaba.otter.canal.admin.model.BaseModel; import com.alibaba.otter.canal.admin.model.BaseModel;
import com.alibaba.otter.canal.admin.model.CanalInstanceConfig; import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
import com.alibaba.otter.canal.admin.model.Pager; import com.alibaba.otter.canal.admin.model.Pager;
@@ -143,8 +143,7 @@ public class CanalInstanceController {
* @return 是否成功 * @return 是否成功
*/ */
@PutMapping(value = "/instance/status/{id}") @PutMapping(value = "/instance/status/{id}")
public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option, public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option, @PathVariable String env) {
@PathVariable String env) {
return BaseModel.getInstance(canalInstanceConfigService.instanceOperation(id, option)); return BaseModel.getInstance(canalInstanceConfigService.instanceOperation(id, option));
} }

34
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/UserController.java
View File

@@ -1,20 +1,28 @@
package com.alibaba.otter.canal.admin.controller; package com.alibaba.otter.canal.admin.controller;
import com.alibaba.otter.canal.admin.service.UserService;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import com.alibaba.otter.canal.admin.model.BaseModel;
import com.alibaba.otter.canal.admin.model.User;
import javax.servlet.http.HttpServletRequest;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.servlet.http.HttpServletRequest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.alibaba.otter.canal.admin.model.BaseModel;
import com.alibaba.otter.canal.admin.model.User;
import com.alibaba.otter.canal.admin.service.UserService;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
/** /**
* 用户管理控制层 * 用户管理控制层
* *
@@ -26,9 +34,9 @@ import java.util.concurrent.TimeUnit;
public class UserController { public class UserController {
public static final LoadingCache<String, User> loginUsers = Caffeine.newBuilder() public static final LoadingCache<String, User> loginUsers = Caffeine.newBuilder()
.maximumSize(10_000) .maximumSize(10_000)
.expireAfterAccess(30, TimeUnit.MINUTES) .expireAfterAccess(30, TimeUnit.MINUTES)
.build(key -> null); // 用户登录信息缓存 .build(key -> null); // 用户登录信息缓存
@Autowired @Autowired
UserService userService; UserService userService;

5
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/handler/CustomExceptionHandler.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.admin.handler; package com.alibaba.otter.canal.admin.handler;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import com.alibaba.otter.canal.admin.model.BaseModel;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
@@ -10,6 +8,9 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.ResponseStatus;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import com.alibaba.otter.canal.admin.model.BaseModel;
/** /**
* MVC异常拦截器 * MVC异常拦截器
* *

3
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalCluster.java
View File

@@ -3,10 +3,11 @@ package com.alibaba.otter.canal.admin.model;
import io.ebean.Finder; import io.ebean.Finder;
import io.ebean.annotation.WhenModified; import io.ebean.annotation.WhenModified;
import java.util.Date;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import java.util.Date;
/** /**
* Canal集群信息实体类 * Canal集群信息实体类

18
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalInstanceConfig.java
View File

@@ -1,12 +1,18 @@
package com.alibaba.otter.canal.admin.model; package com.alibaba.otter.canal.admin.model;
import java.util.Date;
import javax.persistence.*;
import io.ebean.Finder; import io.ebean.Finder;
import io.ebean.annotation.WhenModified; import io.ebean.annotation.WhenModified;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Transient;
/** /**
* Canal实例配置信息实体类 * Canal实例配置信息实体类
* *
@@ -44,14 +50,14 @@ public class CanalInstanceConfig extends Model {
private String name; private String name;
private String content; private String content;
private String contentMd5; private String contentMd5;
private String status; // 1: 正常 0: 停止 private String status; // 1: 正常 0: 停止
@WhenModified @WhenModified
private Date modifiedTime; private Date modifiedTime;
@Transient @Transient
private String clusterServerId; private String clusterServerId;
@Transient @Transient
private String runningStatus = "0"; // 1: 运行中 0: 停止 private String runningStatus = "0"; // 1: 运行中 0: 停止
public void init() { public void init() {
status = "1"; status = "1";

6
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Model.java
View File

@@ -1,13 +1,13 @@
package com.alibaba.otter.canal.admin.model; package com.alibaba.otter.canal.admin.model;
import io.ebean.Ebean;
import io.ebean.EbeanServer;
import javax.persistence.MappedSuperclass; import javax.persistence.MappedSuperclass;
import javax.persistence.OptimisticLockException; import javax.persistence.OptimisticLockException;
import org.apache.commons.beanutils.PropertyUtils; import org.apache.commons.beanutils.PropertyUtils;
import io.ebean.Ebean;
import io.ebean.EbeanServer;
/** /**
* EBean Model扩展类 * EBean Model扩展类
* *

8
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/NodeServer.java
View File

@@ -5,7 +5,13 @@ import io.ebean.annotation.WhenModified;
import java.util.Date; import java.util.Date;
import javax.persistence.*; import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
/** /**
* 节点信息实体类 * 节点信息实体类

3
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/User.java
View File

@@ -3,11 +3,12 @@ package com.alibaba.otter.canal.admin.model;
import io.ebean.Finder; import io.ebean.Finder;
import io.ebean.annotation.WhenCreated; import io.ebean.annotation.WhenCreated;
import java.util.Date;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Transient; import javax.persistence.Transient;
import java.util.Date;
/** /**
* 用户信息实体类 * 用户信息实体类

4
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/NodeServerService.java
View File

@@ -1,10 +1,10 @@
package com.alibaba.otter.canal.admin.service; package com.alibaba.otter.canal.admin.service;
import java.util.List;
import com.alibaba.otter.canal.admin.model.NodeServer; import com.alibaba.otter.canal.admin.model.NodeServer;
import com.alibaba.otter.canal.admin.model.Pager; import com.alibaba.otter.canal.admin.model.Pager;
import java.util.List;
public interface NodeServerService { public interface NodeServerService {
void save(NodeServer nodeServer); void save(NodeServer nodeServer);

4
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java
View File

@@ -109,12 +109,12 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
})); }));
} }
for (Future<Void> f :futures){ for (Future<Void> f : futures) {
try { try {
f.get(3, TimeUnit.SECONDS); f.get(3, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
// ignore // ignore
} catch (TimeoutException e){ } catch (TimeoutException e) {
break; break;
} }
} }

4
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java
View File

@@ -154,12 +154,12 @@ public class NodeServerServiceImpl implements NodeServerService {
return !status; return !status;
})); }));
} }
for (Future<Boolean> f :futures){ for (Future<Boolean> f : futures) {
try { try {
f.get(3, TimeUnit.SECONDS); f.get(3, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
// ignore // ignore
} catch (TimeoutException e){ } catch (TimeoutException e) {
break; break;
} }
} }

45
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java
View File

@@ -6,21 +6,27 @@ import java.util.Map;
public class CommonMessage implements Serializable { public class CommonMessage implements Serializable {
private static final long serialVersionUID = 2611556444074013268L; private static final long serialVersionUID = 2611556444074013268L;
private String database; // 数据库或schema private String database; // 数据库或schema
private String table; // 表名 private String table; // 表名
private List<String> pkNames; private List<String> pkNames;
private Boolean isDdl; private Boolean isDdl;
private String type; // 类型: INSERT UPDATE DELETE private String type; // 类型:
// INSERT
// UPDATE
// DELETE
// binlog executeTime // binlog executeTime
private Long es; // 执行耗时 private Long es; // 执行耗时
// dml build timeStamp // dml build timeStamp
private Long ts; // 同步时间 private Long ts; // 同步时间
private String sql; // 执行的sql, dml sql为空 private String sql; // 执行的sql,
private List<Map<String, Object>> data; // 数据列表 // dml
private List<Map<String, Object>> old; // 旧数据列表, 用于update, size和data的size一一对应 // sql为空
private List<Map<String, Object>> data; // 数据列表
private List<Map<String, Object>> old; // 旧数据列表,
// 用于update,
// size和data的size一一对应
public String getDatabase() { public String getDatabase() {
return database; return database;
@@ -115,17 +121,8 @@ public class CommonMessage implements Serializable {
@Override @Override
public String toString() { public String toString() {
return "CommonMessage{" + return "CommonMessage{" + "database='" + database + '\'' + ", table='" + table + '\'' + ", pkNames=" + pkNames
"database='" + database + '\'' + + ", isDdl=" + isDdl + ", type='" + type + '\'' + ", es=" + es + ", ts=" + ts + ", sql='" + sql + '\''
", table='" + table + '\'' + + ", data=" + data + ", old=" + old + '}';
", pkNames=" + pkNames +
", isDdl=" + isDdl +
", type='" + type + '\'' +
", es=" + es +
", ts=" + ts +
", sql='" + sql + '\'' +
", data=" + data +
", old=" + old +
'}';
} }
} }

20
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java
View File

@@ -1,6 +1,12 @@
package com.alibaba.otter.canal.connector.core.filter; package com.alibaba.otter.canal.connector.core.filter;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -14,16 +20,16 @@ import com.googlecode.aviator.Expression;
*/ */
public class AviaterRegexFilter { public class AviaterRegexFilter {
private static final String SPLIT = ","; private static final String SPLIT = ",";
private static final String PATTERN_SPLIT = "|"; private static final String PATTERN_SPLIT = "|";
private static final String FILTER_EXPRESSION = "regex(pattern,target)"; private static final String FILTER_EXPRESSION = "regex(pattern,target)";
private static final RegexFunction regexFunction = new RegexFunction(); private static final RegexFunction regexFunction = new RegexFunction();
private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true); private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
static { static {
AviatorEvaluator.addFunction(regexFunction); AviatorEvaluator.addFunction(regexFunction);
} }
private static final Comparator<String> COMPARATOR = new StringComparator(); private static final Comparator<String> COMPARATOR = new StringComparator();
final private String pattern; final private String pattern;
final private boolean defaultEmptyValue; final private boolean defaultEmptyValue;

21
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java
View File

@@ -13,15 +13,18 @@ import com.google.common.collect.MigrateMap;
public class PatternUtils { public class PatternUtils {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> { private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(),
try { pattern -> {
PatternCompiler pc = new Perl5Compiler(); try {
return pc.compile(pattern, PatternCompiler pc = new Perl5Compiler();
Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK); return pc.compile(pattern,
} catch (MalformedPatternException e) { Perl5Compiler.CASE_INSENSITIVE_MASK
throw new RuntimeException(e); | Perl5Compiler.READ_ONLY_MASK
} | Perl5Compiler.SINGLELINE_MASK);
}); } catch (MalformedPatternException e) {
throw new RuntimeException(e);
}
});
public static Pattern getPattern(String pattern) { public static Pattern getPattern(String pattern) {
return patterns.get(pattern); return patterns.get(pattern);

2
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java
View File

@@ -5,10 +5,10 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.config.MQProperties; import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;

124
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
View File

@@ -1,9 +1,15 @@
package com.alibaba.otter.canal.connector.core.producer; package com.alibaba.otter.canal.connector.core.producer;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import com.google.common.base.Function;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate; import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
@@ -13,6 +19,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.google.common.base.Function;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker; import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
@@ -28,82 +35,65 @@ import com.google.protobuf.InvalidProtocolBufferException;
public class MQMessageUtils { public class MQMessageUtils {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> { pkHashConfigs -> {
List<PartitionData> datas = Lists List<PartitionData> datas = Lists.newArrayList();
.newArrayList();
String[] pkHashConfigArray = StringUtils String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
.split(StringUtils.replace( ",",
pkHashConfigs, ";"),
",", ";");
";"), ";"); // schema.table:id^name
// schema.table:id^name for (String pkHashConfig : pkHashConfigArray) {
for (String pkHashConfig : pkHashConfigArray) { PartitionData data = new PartitionData();
PartitionData data = new PartitionData(); int i = pkHashConfig.lastIndexOf(":");
int i = pkHashConfig if (i > 0) {
.lastIndexOf(":"); String pkStr = pkHashConfig.substring(i + 1);
if (i > 0) { if (pkStr.equalsIgnoreCase("$pk$")) {
String pkStr = pkHashConfig data.hashMode.autoPkHash = true;
.substring(i + 1); } else {
if (pkStr.equalsIgnoreCase( data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
"$pk$")) { '^'));
data.hashMode.autoPkHash = true; }
} else {
data.hashMode.pkNames = Lists
.newArrayList(
StringUtils
.split(
pkStr,
'^'));
}
pkHashConfig = pkHashConfig pkHashConfig = pkHashConfig.substring(0,
.substring(0, i); i);
} else { } else {
data.hashMode.tableHash = true; data.hashMode.tableHash = true;
} }
if (!isWildCard( if (!isWildCard(pkHashConfig)) {
pkHashConfig)) { data.simpleName = pkHashConfig;
data.simpleName = pkHashConfig; } else {
} else { data.regexFilter = new AviaterRegexFilter(pkHashConfig);
data.regexFilter = new AviaterRegexFilter( }
pkHashConfig); datas.add(data);
} }
datas.add(data);
}
return datas; return datas;
}); });
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() { new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) { public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists List<DynamicTopicData> datas = Lists.newArrayList();
.newArrayList(); String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
String[] dynamicTopicArray = StringUtils ",",
.split(StringUtils.replace( ";"),
pkHashConfigs, ";");
",",
";"), ";");
// schema.table // schema.table
for (String dynamicTopic : dynamicTopicArray) { for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData(); DynamicTopicData data = new DynamicTopicData();
if (!isWildCard( if (!isWildCard(dynamicTopic)) {
dynamicTopic)) {
data.simpleName = dynamicTopic; data.simpleName = dynamicTopic;
} else { } else {
if (dynamicTopic if (dynamicTopic.contains("\\.")) {
.contains("\\.")) { data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
data.tableRegexFilter = new AviaterRegexFilter(
dynamicTopic);
} else { } else {
data.schemaRegexFilter = new AviaterRegexFilter( data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
dynamicTopic);
} }
} }
datas.add(data); datas.add(data);
@@ -242,8 +232,8 @@ public class MQMessageUtils {
* @return 分区message数组 * @return 分区message数组
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs, public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum,
boolean databaseHash) { String pkHashConfigs, boolean databaseHash) {
if (partitionsNum == null) { if (partitionsNum == null) {
partitionsNum = 1; partitionsNum = 1;
} }
@@ -631,8 +621,8 @@ public class MQMessageUtils {
private static boolean isWildCard(String value) { private static boolean isWildCard(String value) {
// not contaiins '.' ? // not contaiins '.' ?
return StringUtils.containsAny(value, return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' }); '^' });
} }
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName, private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,

2
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java
View File

@@ -3,8 +3,8 @@ package com.alibaba.otter.canal.connector.core.spi;
import java.util.Properties; import java.util.Properties;
import com.alibaba.otter.canal.connector.core.config.MQProperties; import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination; import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
/** /**

6
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.connector.core.spi; package com.alibaba.otter.canal.connector.core.spi;
import java.lang.annotation.*; import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/** /**
* SPI装载器注解 * SPI装载器注解

8
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java
View File

@@ -7,7 +7,8 @@ import java.util.Enumeration;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
public class URLClassExtensionLoader extends URLClassLoader { public class URLClassExtensionLoader extends URLClassLoader {
public URLClassExtensionLoader(URL[] urls) {
public URLClassExtensionLoader(URL[] urls){
super(urls); super(urls);
} }
@@ -18,9 +19,8 @@ public class URLClassExtensionLoader extends URLClassLoader {
return c; return c;
} }
if (name.startsWith("java.") || name.startsWith("org.slf4j.") if (name.startsWith("java.") || name.startsWith("org.slf4j.") || name.startsWith("org.apache.logging")
|| name.startsWith("org.apache.logging") || name.startsWith("org.apache.commons.logging.")) {
|| name.startsWith("org.apache.commons.logging.")) {
// || name.startsWith("org.apache.hadoop.")) // || name.startsWith("org.apache.hadoop."))
// { // {
c = super.loadClass(name); c = super.loadClass(name);

1
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java
View File

@@ -7,6 +7,7 @@ package com.alibaba.otter.canal.connector.core.util;
* @version 1.0.0 * @version 1.0.0
*/ */
public interface Callback { public interface Callback {
void commit(); void commit();
void rollback(); void rollback();

9
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java
View File

@@ -2,13 +2,13 @@ package com.alibaba.otter.canal.connector.core.util;
import java.util.List; import java.util.List;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket; import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream; import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat; import com.google.protobuf.WireFormat;
@@ -59,12 +59,11 @@ public class CanalMessageSerializerUtil {
output.checkNoSpaceLeft(); output.checkNoSpaceLeft();
return body; return body;
} else if (!CollectionUtils.isEmpty(data.getEntries())) { } else if (!CollectionUtils.isEmpty(data.getEntries())) {
// mq模式只会走到非rowEntry模式 // mq模式只会走到非rawEntry模式
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
for (CanalEntry.Entry entry : data.getEntries()) { for (CanalEntry.Entry entry : data.getEntries()) {
if (filterTransactionEntry if (filterTransactionEntry
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
continue; continue;
} }
@@ -98,7 +97,7 @@ public class CanalMessageSerializerUtil {
switch (p.getType()) { switch (p.getType()) {
case MESSAGES: { case MESSAGES: {
if (!p.getCompression().equals(CanalPacket.Compression.NONE) if (!p.getCompression().equals(CanalPacket.Compression.NONE)
&& !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) { && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
throw new CanalClientException("compression is not supported in this connector"); throw new CanalClientException("compression is not supported in this connector");
} }

5
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java
View File

@@ -1,12 +1,13 @@
package com.alibaba.otter.canal.connector.core.util; package com.alibaba.otter.canal.connector.core.util;
import java.util.Date;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
import java.util.Date;
public class DateUtil { public class DateUtil {
private static DateTimeZone dateTimeZone; private static DateTimeZone dateTimeZone;
static { static {

11
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java
View File

@@ -2,7 +2,12 @@ package com.alibaba.otter.canal.connector.core.util;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
import java.sql.*; import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -72,8 +77,8 @@ public class JdbcTypeUtil {
} }
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) { public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR if (value == null
|| sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) { || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
return null; return null;
} }

34
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java
View File

@@ -1,6 +1,11 @@
package com.alibaba.otter.canal.connector.core.util; package com.alibaba.otter.canal.connector.core.util;
import java.util.*; import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
@@ -22,7 +27,7 @@ public class MessageUtil {
List<CommonMessage> msgs = new ArrayList<>(entries.size()); List<CommonMessage> msgs = new ArrayList<>(entries.size());
for (CanalEntry.Entry entry : entries) { for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue; continue;
} }
@@ -31,7 +36,7 @@ public class MessageUtil {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e); e);
} }
CanalEntry.EventType eventType = rowChange.getEventType(); CanalEntry.EventType eventType = rowChange.getEventType();
@@ -55,7 +60,7 @@ public class MessageUtil {
int i = 0; int i = 0;
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) { for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) { && eventType != CanalEntry.EventType.DELETE) {
continue; continue;
} }
@@ -78,11 +83,11 @@ public class MessageUtil {
row.put(column.getName(), null); row.put(column.getName(), null);
} else { } else {
row.put(column.getName(), row.put(column.getName(),
JdbcTypeUtil.typeConvert(msg.getTable(), JdbcTypeUtil.typeConvert(msg.getTable(),
column.getName(), column.getName(),
column.getValue(), column.getValue(),
column.getSqlType(), column.getSqlType(),
column.getMysqlType())); column.getMysqlType()));
} }
// 获取update为true的字段 // 获取update为true的字段
if (column.getUpdated()) { if (column.getUpdated()) {
@@ -100,12 +105,11 @@ public class MessageUtil {
if (column.getIsNull()) { if (column.getIsNull()) {
rowOld.put(column.getName(), null); rowOld.put(column.getName(), null);
} else { } else {
rowOld.put(column.getName(), rowOld.put(column.getName(), JdbcTypeUtil.typeConvert(msg.getTable(),
JdbcTypeUtil.typeConvert(msg.getTable(), column.getName(),
column.getName(), column.getValue(),
column.getValue(), column.getSqlType(),
column.getSqlType(), column.getMysqlType()));
column.getMysqlType()));
} }
} }
} }

4
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java
View File

@@ -1,8 +1,8 @@
package com.alibaba.otter.canal.connector.core.util; package com.alibaba.otter.canal.connector.core.util;
public class TimeZone { public class TimeZone {
public final static String LOCATION_TIME_ZONE; // 当前时区
public final static String LOCATION_TIME_ZONE; // 当前时区
static { static {
java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault(); java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault();

2
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java
View File

@@ -15,7 +15,7 @@ public class KafkaProducerConfig extends MQProperties {
private Map<String, Object> kafkaProperties = new LinkedHashMap<>(); private Map<String, Object> kafkaProperties = new LinkedHashMap<>();
private boolean kerberosEnabled = false; private boolean kerberosEnabled = false;
private String krb5File; private String krb5File;
private String jaasFile; private String jaasFile;

19
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java
View File

@@ -1,20 +1,25 @@
package com.alibaba.otter.canal.connector.kafka.consumer; package com.alibaba.otter.canal.connector.kafka.consumer;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringDeserializer;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.MessageUtil; import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
@@ -72,8 +77,7 @@ public class CanalKafkaConsumer implements CanalMsgConsumer {
@Override @Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) { public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
if (!flatMessage) { if (!flatMessage) {
ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer.poll(unit.toMillis(timeout));
.poll(unit.toMillis(timeout));
if (!records.isEmpty()) { if (!records.isEmpty()) {
currentOffsets.clear(); currentOffsets.clear();
List<CommonMessage> messages = new ArrayList<>(); List<CommonMessage> messages = new ArrayList<>();
@@ -86,8 +90,7 @@ public class CanalKafkaConsumer implements CanalMsgConsumer {
return messages; return messages;
} }
} else { } else {
ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer.poll(unit.toMillis(timeout));
.poll(unit.toMillis(timeout));
if (!records.isEmpty()) { if (!records.isEmpty()) {
List<CommonMessage> messages = new ArrayList<>(); List<CommonMessage> messages = new ArrayList<>();

2
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java
View File

@@ -2,9 +2,9 @@ package com.alibaba.otter.canal.connector.kafka.consumer;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Deserializer;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
/** /**

26
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java
View File

@@ -1,12 +1,14 @@
package com.alibaba.otter.canal.connector.kafka.producer; package com.alibaba.otter.canal.connector.kafka.producer;
import java.io.File; import java.io.File;
import java.util.*; import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.Producer;
@@ -19,13 +21,14 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate; import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer; import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination; import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils; import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData; import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI; import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants;
import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig; import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
@@ -126,8 +129,9 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
List result; List result;
if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) { if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) {
// 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化 // 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
Map<String, Message> messageMap = MQMessageUtils Map<String, Message> messageMap = MQMessageUtils.messageTopics(message,
.messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic()); mqDestination.getTopic(),
mqDestination.getDynamicTopic());
// 针对不同的topic,引入多线程提升效率 // 针对不同的topic,引入多线程提升效率
for (Map.Entry<String, Message> entry : messageMap.entrySet()) { for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
@@ -222,18 +226,14 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i]; FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) { if (flatMessagePart != null) {
records.add(new ProducerRecord<>(topicName, records.add(new ProducerRecord<>(topicName, i, null, JSON.toJSONBytes(flatMessagePart,
i, SerializerFeature.WriteMapNullValue)));
null,
JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
} }
} }
} else { } else {
final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0; final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
records.add(new ProducerRecord<>(topicName, records.add(new ProducerRecord<>(topicName, partition, null, JSON.toJSONBytes(flatMessage,
partition, SerializerFeature.WriteMapNullValue)));
null,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
} }
} }
} }

4
connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java
View File

@@ -1,9 +1,9 @@
package com.alibaba.otter.canal.connector.kafka.producer; package com.alibaba.otter.canal.connector.kafka.producer;
import org.apache.kafka.common.serialization.Serializer;
import java.util.Map; import java.util.Map;
import org.apache.kafka.common.serialization.Serializer;
/** /**
* Kafka Message类的序列化 * Kafka Message类的序列化
* *

5
connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java
View File

@@ -20,8 +20,9 @@ public class CanalKafkaProducerTest {
pro.load(in); pro.load(in);
ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class); ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
CanalMQProducer canalMQProducer = loader CanalMQProducer canalMQProducer = loader.getExtension("kafka",
.getExtension("kafka", "/../../deployer/target/canal/plugin", "/../../deployer/target/canal/plugin"); "/../../deployer/target/canal/plugin",
"/../../deployer/target/canal/plugin");
canalMQProducer.init(pro); canalMQProducer.init(pro);
in.close(); in.close();

15
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java
View File

@@ -8,7 +8,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -16,6 +15,7 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants; import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.core.util.MessageUtil; import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants; import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
@@ -23,7 +23,13 @@ import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProv
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException; import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.rabbitmq.client.*; import com.rabbitmq.client.AMQP;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.Consumer;
import com.rabbitmq.client.DefaultConsumer;
import com.rabbitmq.client.Envelope;
/** /**
* RabbitMQ consumer SPI 实现 * RabbitMQ consumer SPI 实现
@@ -34,8 +40,7 @@ import com.rabbitmq.client.*;
@SPI("rabbitmq") @SPI("rabbitmq")
public class CanalRabbitMQConsumer implements CanalMsgConsumer { public class CanalRabbitMQConsumer implements CanalMsgConsumer {
private static final Logger logger = LoggerFactory private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQConsumer.class);
.getLogger(CanalRabbitMQConsumer.class);
// 链接地址 // 链接地址
private String nameServer; private String nameServer;
@@ -131,7 +136,7 @@ public class CanalRabbitMQConsumer implements CanalMsgConsumer {
CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class); CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
messageList.add(commonMessage); messageList.add(commonMessage);
} }
ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList); ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
try { try {
messageBlockingQueue.put(batchMessage); messageBlockingQueue.put(batchMessage);
} catch (InterruptedException e) { } catch (InterruptedException e) {

4
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java
View File

@@ -7,8 +7,8 @@ import java.util.concurrent.TimeUnit;
public class ConsumerBatchMessage<T> { public class ConsumerBatchMessage<T> {
private final List<T> data; private final List<T> data;
private CountDownLatch latch; private CountDownLatch latch;
private boolean hasFailure = false; private boolean hasFailure = false;
public ConsumerBatchMessage(List<T> data){ public ConsumerBatchMessage(List<T> data){
this.data = data; this.data = data;

3
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java
View File

@@ -7,8 +7,7 @@ import com.alibaba.mq.amqp.utils.UserUtils;
import com.rabbitmq.client.impl.CredentialsProvider; import com.rabbitmq.client.impl.CredentialsProvider;
/** /**
* aliyun amqp协议 账号类 * aliyun amqp协议 账号类 暂不支持STS授权情况
* 暂不支持STS授权情况
* *
* @version 1.0.0 * @version 1.0.0
*/ */

7
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java
View File

@@ -15,11 +15,11 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException; import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate; import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer; import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination; import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils; import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI; import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants; import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig; import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
@@ -102,8 +102,9 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
try { try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) { if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic // 动态topic
Map<String, Message> messageMap = MQMessageUtils Map<String, Message> messageMap = MQMessageUtils.messageTopics(message,
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic()); destination.getTopic(),
destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) { for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
final String topicName = entry.getKey().replace('.', '_'); final String topicName = entry.getKey().replace('.', '_');

3
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java
View File

@@ -41,8 +41,7 @@ import com.google.common.collect.Lists;
@SPI("rocketmq") @SPI("rocketmq")
public class CanalRocketMQConsumer implements CanalMsgConsumer { public class CanalRocketMQConsumer implements CanalMsgConsumer {
private static final Logger logger = LoggerFactory private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQConsumer.class);
.getLogger(CanalRocketMQConsumer.class);
private static final String CLOUD_ACCESS_CHANNEL = "cloud"; private static final String CLOUD_ACCESS_CHANNEL = "cloud";
private String nameServer; private String nameServer;

4
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java
View File

@@ -7,8 +7,8 @@ import java.util.concurrent.TimeUnit;
public class ConsumerBatchMessage<T> { public class ConsumerBatchMessage<T> {
private final List<T> data; private final List<T> data;
private CountDownLatch latch; private CountDownLatch latch;
private boolean hasFailure = false; private boolean hasFailure = false;
public ConsumerBatchMessage(List<T> data){ public ConsumerBatchMessage(List<T> data){
this.data = data; this.data = data;

30
connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java
View File

@@ -26,11 +26,11 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalException; import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate; import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer; import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.producer.MQDestination; import com.alibaba.otter.canal.connector.core.producer.MQDestination;
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils; import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer; import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import com.alibaba.otter.canal.connector.core.spi.SPI; import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.Callback;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil; import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants; import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig; import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
@@ -58,8 +58,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
loadRocketMQProperties(properties); loadRocketMQProperties(properties);
RPCHook rpcHook = null; RPCHook rpcHook = null;
if (mqProperties.getAliyunAccessKey().length() > 0 if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0) {
&& mqProperties.getAliyunSecretKey().length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials(); SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey()); sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey()); sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
@@ -126,8 +125,9 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
try { try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) { if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic // 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic()); destination.getTopic(),
destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) { for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_'); String topicName = entry.getKey().replace('.', '_');
@@ -155,8 +155,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
} }
} }
public void send(final MQDestination destination, String topicName, public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
com.alibaba.otter.canal.protocol.Message message) {
if (!mqProperties.isFlatMessage()) { if (!mqProperties.isFlatMessage()) {
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) { if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
// 并发构造 // 并发构造
@@ -175,9 +174,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
if (dataPartition != null) { if (dataPartition != null) {
final int index = i; final int index = i;
template.submit(() -> { template.submit(() -> {
Message data = new Message(topicName, Message data = new Message(topicName, CanalMessageSerializerUtil.serializer(dataPartition,
CanalMessageSerializerUtil.serializer(dataPartition, mqProperties.isFilterTransactionEntry()));
mqProperties.isFilterTransactionEntry()));
sendMessage(data, index); sendMessage(data, index);
}); });
} }
@@ -186,8 +184,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
template.waitForResult(); template.waitForResult();
} else { } else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0; final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
Message data = new Message(topicName, Message data = new Message(topicName, CanalMessageSerializerUtil.serializer(message,
CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry())); mqProperties.isFilterTransactionEntry()));
sendMessage(data, partition); sendMessage(data, partition);
} }
} else { } else {
@@ -220,8 +218,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
final int index = i; final int index = i;
template.submit(() -> { template.submit(() -> {
List<Message> messages = flatMessagePart.stream() List<Message> messages = flatMessagePart.stream()
.map(flatMessage -> new Message(topicName, .map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))) SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList()); .collect(Collectors.toList());
// 批量发送 // 批量发送
sendMessage(messages, index); sendMessage(messages, index);
@@ -234,8 +232,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
} else { } else {
final int partition = destination.getPartition() != null ? destination.getPartition() : 0; final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
List<Message> messages = flatMessages.stream() List<Message> messages = flatMessages.stream()
.map(flatMessage -> new Message(topicName, .map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))) SerializerFeature.WriteMapNullValue)))
.collect(Collectors.toList()); .collect(Collectors.toList());
// 批量发送 // 批量发送
sendMessage(messages, partition); sendMessage(messages, partition);

13
connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java
View File

@@ -1,5 +1,11 @@
package com.alibaba.otter.canal.connector.tcp.consumer; package com.alibaba.otter.canal.connector.tcp.consumer;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy; import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
@@ -12,13 +18,6 @@ import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.tcp.config.TCPConstants; import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
/** /**
* TCP 消费者连接器, 一个destination对应一个SPI实例 * TCP 消费者连接器, 一个destination对应一个SPI实例
* *