chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1 from alibaba/master

Browse Source 
update
This commit is contained in:
bigbro
2019-01-29 16:03:43 +08:00
committed by GitHub
parent 47d7914304 32031da9db
commit ccd39e9d62
27 changed files with 1102 additions and 868 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/README.md
View File

@@ -55,7 +55,7 @@ canal.conf:
hbase.zookeeper.quorum: slave1 hbase.zookeeper.quorum: slave1
hbase.zookeeper.property.clientPort: 2181 hbase.zookeeper.property.clientPort: 2181
zookeeper.znode.parent: /hbase zookeeper.znode.parent: /hbase
mqTopics: # MQ topic租, 如果是kafka或者rockeMQ模式可配置此项, 与canalInstances不能并存 mqTopics: # MQ topic组, 如果是kafka或者rockeMQ模式可配置此项, 与canalInstances不能并存
- mqMode: kafka # MQ的模式: kafak/rocketMQ - mqMode: kafka # MQ的模式: kafak/rocketMQ
topic: example # MQ topic topic: example # MQ topic
groups: # group组 groups: # group组

41
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/BootstrapConfiguration.java
View File

@@ -1,14 +1,14 @@
package com.alibaba.otter.canal.adapter.launcher.config; package com.alibaba.otter.canal.adapter.launcher.config;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import com.alibaba.otter.canal.adapter.launcher.monitor.AdapterRemoteConfigMonitor;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment; import org.springframework.core.env.Environment;
import com.alibaba.otter.canal.adapter.launcher.monitor.remote.RemoteConfigLoader;
import com.alibaba.otter.canal.adapter.launcher.monitor.remote.RemoteConfigLoaderFactory;
/** /**
* Bootstrap级别配置加载 * Bootstrap级别配置加载
* *
@@ -17,28 +17,25 @@ import org.springframework.core.env.Environment;
*/ */
public class BootstrapConfiguration { public class BootstrapConfiguration {
private static final Logger logger = LoggerFactory.getLogger(BootstrapConfiguration.class);
@Autowired @Autowired
private Environment env; private Environment env;
private RemoteConfigLoader remoteConfigLoader = null;
@PostConstruct @PostConstruct
public void loadRemoteConfig() { public void loadRemoteConfig() {
try { remoteConfigLoader = RemoteConfigLoaderFactory.getRemoteConfigLoader(env);
// 加载远程配置 if (remoteConfigLoader != null) {
String jdbcUrl = env.getProperty("canal.manager.jdbc.url"); remoteConfigLoader.loadRemoteConfig();
if (StringUtils.isNotEmpty(jdbcUrl)) { remoteConfigLoader.loadRemoteAdapterConfigs();
String jdbcUsername = env.getProperty("canal.manager.jdbc.username"); remoteConfigLoader.startMonitor(); // 启动监听
String jdbcPassword = env.getProperty("canal.manager.jdbc.password"); }
AdapterRemoteConfigMonitor configMonitor = new AdapterRemoteConfigMonitor(jdbcUrl, }
jdbcUsername,
jdbcPassword); @PreDestroy
configMonitor.loadRemoteConfig(); public synchronized void destroy() {
configMonitor.loadRemoteAdapterConfigs(); if (remoteConfigLoader != null) {
configMonitor.start(); // 启动监听 remoteConfigLoader.destroy();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} }
} }
} }

23
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java
View File

@@ -4,8 +4,6 @@ import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import javax.annotation.Resource; import javax.annotation.Resource;
import com.alibaba.otter.canal.adapter.launcher.monitor.AdapterRemoteConfigMonitor;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -29,27 +27,25 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
@RefreshScope @RefreshScope
public class CanalAdapterService { public class CanalAdapterService {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterService.class); private static final Logger logger = LoggerFactory.getLogger(CanalAdapterService.class);
private CanalAdapterLoader adapterLoader; private CanalAdapterLoader adapterLoader;
@Resource @Resource
private ContextRefresher contextRefresher; private ContextRefresher contextRefresher;
@Resource @Resource
private AdapterCanalConfig adapterCanalConfig; private AdapterCanalConfig adapterCanalConfig;
@Resource @Resource
private Environment env; private Environment env;
// 注入bean保证优先注册 // 注入bean保证优先注册
@Resource @Resource
private SpringContext springContext; private SpringContext springContext;
@Resource @Resource
private SyncSwitch syncSwitch; private SyncSwitch syncSwitch;
private volatile boolean running = false; private volatile boolean running = false;
private AdapterRemoteConfigMonitor configMonitor = null;
@PostConstruct @PostConstruct
public synchronized void init() { public synchronized void init() {
@@ -75,9 +71,6 @@ public class CanalAdapterService {
try { try {
running = false; running = false;
logger.info("## stop the canal client adapters"); logger.info("## stop the canal client adapters");
if (configMonitor != null) {
configMonitor.destroy();
}
if (adapterLoader != null) { if (adapterLoader != null) {
adapterLoader.destroy(); adapterLoader.destroy();

56
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/ConfigItem.java Normal file
View File

@@ -0,0 +1,56 @@
package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
/**
* 配置对应对象
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public class ConfigItem {
private Long id;
private String category;
private String name;
private String content;
private long modifiedTime;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getCategory() {
return category;
}
public void setCategory(String category) {
this.category = category;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(long modifiedTime) {
this.modifiedTime = modifiedTime;
}
}

259
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/AdapterRemoteConfigMonitor.java → client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -1,10 +1,11 @@
package com.alibaba.otter.canal.adapter.launcher.monitor; package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter; import java.io.FileWriter;
import java.net.URL; import java.sql.Connection;
import java.sql.*; import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -13,54 +14,61 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.support.Constant; import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.collect.MapMaker; import com.google.common.collect.MapMaker;
/** /**
* 远程配置装载、监控类 * 基于数据库的远程配置装载器
* *
* @author rewerma @ 2019-01-05 * @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0 * @version 1.0.0
*/ */
public class AdapterRemoteConfigMonitor { public class DbRemoteConfigLoader implements RemoteConfigLoader {
private static final Logger logger = LoggerFactory.getLogger(AdapterRemoteConfigMonitor.class); private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
private Connection conn; private DruidDataSource dataSource;
private String jdbcUrl;
private String jdbcUsername;
private String jdbcPassword;
private long currentConfigTimestamp = 0; private static volatile long currentConfigTimestamp = 0;
private Map<String, ConfigItem> remoteAdapterConfigs = new MapMaker().makeMap(); private Map<String, ConfigItem> remoteAdapterConfigs = new MapMaker().makeMap();
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2, private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
new NamedThreadFactory("remote-adapter-config-scan")); new NamedThreadFactory("remote-adapter-config-scan"));
public AdapterRemoteConfigMonitor(String jdbcUrl, String jdbcUsername, String jdbcPassword){ private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl();
this.jdbcUrl = jdbcUrl;
this.jdbcUsername = jdbcUsername;
this.jdbcPassword = jdbcPassword;
}
private Connection getConn() throws Exception { public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
if (conn == null || conn.isClosed()) { dataSource = new DruidDataSource();
Class.forName("com.mysql.jdbc.Driver"); if (StringUtils.isEmpty(driverName)) {
conn = DriverManager.getConnection(jdbcUrl, jdbcUsername, jdbcPassword); driverName = "com.mysql.jdbc.Driver";
}
dataSource.setDriverClassName(driverName);
dataSource.setUrl(jdbcUrl);
dataSource.setUsername(jdbcUsername);
dataSource.setPassword(jdbcPassword);
dataSource.setInitialSize(1);
dataSource.setMinIdle(1);
dataSource.setMaxActive(1);
dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000);
try {
dataSource.init();
} catch (SQLException e) {
throw new RuntimeException(e.getMessage(), e);
} }
return conn;
} }
/** /**
* 加载远程application.yml配置到本地 * 加载远程application.yml配置
*/ */
@Override
public void loadRemoteConfig() { public void loadRemoteConfig() {
try { try {
// 加载远程adapter配置 // 加载远程adapter配置
@@ -84,7 +92,9 @@ public class AdapterRemoteConfigMonitor {
*/ */
private ConfigItem getRemoteAdapterConfig() { private ConfigItem getRemoteAdapterConfig() {
String sql = "select name, content, modified_time from canal_config where id=2"; String sql = "select name, content, modified_time from canal_config where id=2";
try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) { try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) { if (rs.next()) {
ConfigItem configItem = new ConfigItem(); ConfigItem configItem = new ConfigItem();
configItem.setId(2L); configItem.setId(2L);
@@ -114,54 +124,29 @@ public class AdapterRemoteConfigMonitor {
} }
/** /**
* 启动监听数据库变化 * 加载adapter配置
*/
public void start() {
// 监听application.yml变化
executor.scheduleWithFixedDelay(() -> {
try {
loadRemoteConfig();
} catch (Throwable e) {
logger.error("scan remote application.yml failed", e);
}
}, 10, 3, TimeUnit.SECONDS);
// 监听adapter变化
executor.scheduleWithFixedDelay(() -> {
try {
loadRemoteAdapterConfigs();
} catch (Throwable e) {
logger.error("scan remote adapter configs failed", e);
}
}, 10, 3, TimeUnit.SECONDS);
}
/**
* 加载adapter配置到本地
*/ */
@Override
public void loadRemoteAdapterConfigs() { public void loadRemoteAdapterConfigs() {
try { try {
// 加载远程adapter配置 // 加载远程adapter配置
Map<String, ConfigItem>[] modifiedConfigs = getModifiedAdapterConfigs(); loadModifiedAdapterConfigs();
if (modifiedConfigs != null) {
overrideLocalAdapterConfigs(modifiedConfigs);
}
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
/** /**
* 获取有变动的adapter配置 * 加载有变动的adapter配置
*
* @return Map[0]: 新增修改的配置, Map[1]: 删除的配置
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private Map<String, ConfigItem>[] getModifiedAdapterConfigs() { private void loadModifiedAdapterConfigs() {
Map<String, ConfigItem>[] res = new Map[2]; Map<String, ConfigItem>[] res = new Map[2];
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>(); Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
String sql = "select id, category, name, modified_time from canal_adapter_config"; String sql = "select id, category, name, modified_time from canal_adapter_config";
try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) { try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) { while (rs.next()) {
ConfigItem configItem = new ConfigItem(); ConfigItem configItem = new ConfigItem();
configItem.setId(rs.getLong("id")); configItem.setId(rs.getLong("id"));
@@ -172,7 +157,6 @@ public class AdapterRemoteConfigMonitor {
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
return null;
} }
if (!remoteConfigStatus.isEmpty()) { if (!remoteConfigStatus.isEmpty()) {
@@ -192,10 +176,11 @@ public class AdapterRemoteConfigMonitor {
} }
} }
if (!changedIds.isEmpty()) { if (!changedIds.isEmpty()) {
Map<String, ConfigItem> changedAdapterConfig = new HashMap<>();
String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in (" String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in ("
+ Joiner.on(",").join(changedIds) + ")"; + Joiner.on(",").join(changedIds) + ")";
try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(contentsSql)) { try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(contentsSql)) {
while (rs.next()) { while (rs.next()) {
ConfigItem configItemNew = new ConfigItem(); ConfigItem configItemNew = new ConfigItem();
configItemNew.setId(rs.getLong("id")); configItemNew.setId(rs.getLong("id"));
@@ -206,63 +191,20 @@ public class AdapterRemoteConfigMonitor {
remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(), remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(),
configItemNew); configItemNew);
changedAdapterConfig.put(configItemNew.getCategory() + "/" + configItemNew.getName(), remoteAdapterMonitor.onModify(configItemNew);
configItemNew);
} }
res[0] = changedAdapterConfig.isEmpty() ? null : changedAdapterConfig;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
} }
Map<String, ConfigItem> removedAdapterConfig = new HashMap<>();
for (ConfigItem configItem : remoteAdapterConfigs.values()) { for (ConfigItem configItem : remoteAdapterConfigs.values()) {
if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) { if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) {
// 删除 // 删除
remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName()); remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName());
removedAdapterConfig.put(configItem.getCategory() + "/" + configItem.getName(), null); remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName());
}
}
res[1] = removedAdapterConfig.isEmpty() ? null : removedAdapterConfig;
if (res[0] == null && res[1] == null) {
return null;
} else {
return res;
}
}
/**
* 覆盖adapter配置到本地
*
* @param modifiedAdapterConfigs 变动的配置集合
*/
private void overrideLocalAdapterConfigs(Map<String, ConfigItem>[] modifiedAdapterConfigs) {
Map<String, ConfigItem> changedAdapterConfigs = modifiedAdapterConfigs[0];
if (changedAdapterConfigs != null) {
for (ConfigItem configItem : changedAdapterConfigs.values()) {
try (FileWriter writer = new FileWriter(
getConfPath() + configItem.getCategory() + "/" + configItem.getName())) {
writer.write(configItem.getContent());
writer.flush();
logger.info("## Loaded remote adapter config: {}/{}",
configItem.getCategory(),
configItem.getName());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
Map<String, ConfigItem> removedAdapterConfigs = modifiedAdapterConfigs[1];
if (removedAdapterConfigs != null) {
for (String name : removedAdapterConfigs.keySet()) {
File file = new File(getConfPath() + name);
if (file.exists()) {
deleteDir(file);
logger.info("## Deleted and reloaded remote adapter config: {}", name);
}
} }
} }
} }
@@ -302,66 +244,69 @@ public class AdapterRemoteConfigMonitor {
} }
} }
public void destroy() { /**
executor.shutdownNow(); * 启动监听数据库变化
if (conn != null) { */
@Override
public void startMonitor() {
// 监听application.yml变化
executor.scheduleWithFixedDelay(() -> {
try { try {
conn.close(); loadRemoteConfig();
} catch (SQLException e) { } catch (Throwable e) {
logger.error(e.getMessage(), e); logger.error("scan remote application.yml failed", e);
} }
} }, 10, 3, TimeUnit.SECONDS);
// 监听adapter变化
executor.scheduleWithFixedDelay(() -> {
try {
loadRemoteAdapterConfigs();
} catch (Throwable e) {
logger.error("scan remote adapter configs failed", e);
}
}, 10, 3, TimeUnit.SECONDS);
} }
/** /**
* 配置对应对象 * 销毁
*/ */
public static class ConfigItem { @Override
public void destroy() {
executor.shutdownNow();
try {
dataSource.close();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private Long id; private class RemoteAdapterMonitorImpl implements RemoteAdapterMonitor {
private String category;
private String name;
private String content;
private long modifiedTime;
public Long getId() { @Override
return id; public void onAdd(ConfigItem configItem) {
this.onModify(configItem);
} }
public void setId(Long id) { @Override
this.id = id; public void onModify(ConfigItem configItem) {
try (FileWriter writer = new FileWriter(
getConfPath() + configItem.getCategory() + "/" + configItem.getName())) {
writer.write(configItem.getContent());
writer.flush();
logger.info("## Loaded remote adapter config: {}/{}", configItem.getCategory(), configItem.getName());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
} }
public String getCategory() { @Override
return category; public void onDelete(String name) {
} File file = new File(getConfPath() + name);
if (file.exists()) {
public void setCategory(String category) { deleteDir(file);
this.category = category; logger.info("## Deleted and reloaded remote adapter config: {}", name);
} }
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(long modifiedTime) {
this.modifiedTime = modifiedTime;
} }
} }
} }

31
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitor.java Normal file
View File

@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
/**
* 远程配置监听器接口
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public interface RemoteAdapterMonitor {
/**
* 新增配置事件
*
* @param configItem 配置项
*/
void onAdd(ConfigItem configItem);
/**
* 修改配置事件
*
* @param configItem 配置项
*/
void onModify(ConfigItem configItem);
/**
* 删除配置事件
*
* @param name 配置名
*/
void onDelete(String name);
}

30
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoader.java Normal file
View File

@@ -0,0 +1,30 @@
package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
/**
* 远程配置装载器接口
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public interface RemoteConfigLoader {
/**
* 加载远程application.yml配置到本地
*/
void loadRemoteConfig();
/**
* 加载adapter配置
*/
void loadRemoteAdapterConfigs();
/**
* 启动监听数据库变化
*/
void startMonitor();
/**
* 销毁
*/
void destroy();
}

34
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteConfigLoaderFactory.java Normal file
View File

@@ -0,0 +1,34 @@
package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
/**
* 远程配置装载器工厂类
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public class RemoteConfigLoaderFactory {
private static final Logger logger = LoggerFactory.getLogger(RemoteConfigLoaderFactory.class);
public static RemoteConfigLoader getRemoteConfigLoader(Environment env) {
try {
String jdbcUrl = env.getProperty("canal.manager.jdbc.url");
if (!StringUtils.isEmpty(jdbcUrl)) {
// load remote config
String driverName = env.getProperty("canal.manager.jdbc.driverName");
String jdbcUsername = env.getProperty("canal.manager.jdbc.username");
String jdbcPassword = env.getProperty("canal.manager.jdbc.password");
return new DbRemoteConfigLoader(driverName, jdbcUrl, jdbcUsername, jdbcPassword);
}
// 可扩展其它远程配置加载器
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return null;
}
}

2
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java
View File

@@ -21,7 +21,7 @@ public class ConfigLoader {
private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
/** /**
* 加载HBase表映射配置 * 加载RDB表映射配置
* *
* @return 配置名/配置文件名--对象 * @return 配置名/配置文件名--对象
*/ */

66
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -88,44 +88,46 @@ public class RdbSyncService {
* @param function 回调方法 * @param function 回调方法
*/ */
public void sync(List<Dml> dmls, Function<Dml, Boolean> function) { public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
boolean toExecute = false; try {
for (Dml dml : dmls) { boolean toExecute = false;
if (!toExecute) { for (Dml dml : dmls) {
toExecute = function.apply(dml); if (!toExecute) {
} else { toExecute = function.apply(dml);
function.apply(dml); } else {
function.apply(dml);
}
} }
} if (toExecute) {
if (toExecute) { List<Future> futures = new ArrayList<>();
List<Future> futures = new ArrayList<>(); for (int i = 0; i < threads; i++) {
for (int i = 0; i < threads; i++) { int j = i;
int j = i; futures.add(executorThreads[i].submit(() -> {
futures.add(executorThreads[i].submit(() -> { try {
dmlsPartition[j]
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
batchExecutors[j].rollback();
throw new RuntimeException(e);
}
}));
}
futures.forEach(future -> {
try { try {
dmlsPartition[j] future.get();
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); } catch (ExecutionException | InterruptedException e) {
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
batchExecutors[j].rollback();
throw new RuntimeException(e); throw new RuntimeException(e);
} }
})); });
} }
} finally {
futures.forEach(future -> { for (BatchExecutor batchExecutor : batchExecutors) {
try { if (batchExecutor != null) {
future.get(); batchExecutor.close();
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
} }
});
}
for (BatchExecutor batchExecutor : batchExecutors) {
if (batchExecutor != null) {
batchExecutor.close();
} }
} }
} }

6
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -1,8 +1,10 @@
package com.alibaba.otter.canal.client.kafka; package com.alibaba.otter.canal.client.kafka;
import java.util.*; import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.google.common.collect.MapMaker;
import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -31,7 +33,7 @@ import com.google.common.collect.Lists;
public class KafkaCanalConnector implements CanalMQConnector { public class KafkaCanalConnector implements CanalMQConnector {
protected KafkaConsumer<String, Message> kafkaConsumer; protected KafkaConsumer<String, Message> kafkaConsumer;
protected KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费 protected KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费
protected String topic; protected String topic;
protected Integer partition; protected Integer partition;
protected Properties properties; protected Properties properties;
@@ -39,7 +41,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
protected volatile boolean running = false; protected volatile boolean running = false;
protected boolean flatMessage; protected boolean flatMessage;
private Map<Integer, Long> currentOffsets = new HashMap<>(); private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize, public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
boolean flatMessage){ boolean flatMessage){

1
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
View File

@@ -48,6 +48,7 @@ public class CanalConstants {
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";

149
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -140,80 +140,81 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port); final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData); ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() { ServerRunningMonitors
.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) { public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination); runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() { runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter() { public void processActiveEnter() {
try { try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination); embededCanalServer.start(destination);
} finally { } finally {
MDC.remove(CanalConstants.MDC_DESTINATION); MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
} }
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
} }
}
public void processStop() { public void processActiveExit() {
try { try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) { embededCanalServer.stop(destination);
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":" } finally {
+ port); MDC.remove(CanalConstants.MDC_DESTINATION);
releaseCid(path);
} }
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
} }
}
}); public void processStart() {
if (zkclientx != null) { try {
runningMonitor.setZkClient(zkclientx); if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
} }
// 触发创建一下cid节点 }));
runningMonitor.init();
return runningMonitor;
}
}));
// 初始化monitor机制 // 初始化monitor机制
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
@@ -265,7 +266,8 @@ public class CanalController {
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() { instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode) { public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL)); int scanInterval = Integer
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) { if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -373,7 +375,8 @@ public class CanalController {
InstanceConfig oldConfig = instanceConfigs.put(destination, config); InstanceConfig oldConfig = instanceConfigs.put(destination, config);
if (oldConfig != null) { if (oldConfig != null) {
logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config); logger
.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
} }
} }
} }
@@ -478,6 +481,7 @@ public class CanalController {
} }
public void stop() throws Throwable { public void stop() throws Throwable {
if (canalServer != null) { if (canalServer != null) {
canalServer.stop(); canalServer.stop();
} }
@@ -504,6 +508,17 @@ public class CanalController {
zkclientx.close(); zkclientx.close();
} }
//关闭时清理缓存
if (instanceConfigs != null) {
instanceConfigs.clear();
}
if (managerClients != null) {
managerClients.clear();
}
if (instanceConfigMonitors != null) {
instanceConfigMonitors.clear();
}
ZkClientx.clearClients(); ZkClientx.clearClients();
} }

31
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -7,7 +7,9 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.deployer.monitor.ManagerRemoteConfigMonitor; import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoader;
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoaderFactory;
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteCanalConfigMonitor;
/** /**
* canal独立版本启动的入口类 * canal独立版本启动的入口类
@@ -30,7 +32,7 @@ public class CanalLauncher {
logger.info("## load canal configurations"); logger.info("## load canal configurations");
String conf = System.getProperty("canal.conf", "classpath:canal.properties"); String conf = System.getProperty("canal.conf", "classpath:canal.properties");
Properties properties = new Properties(); Properties properties = new Properties();
ManagerRemoteConfigMonitor managerDbConfigMonitor = null; RemoteConfigLoader remoteConfigLoader = null;
if (conf.startsWith(CLASSPATH_URL_PREFIX)) { if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf)); properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf));
@@ -38,31 +40,24 @@ public class CanalLauncher {
properties.load(new FileInputStream(conf)); properties.load(new FileInputStream(conf));
} }
String jdbcUrl = properties.getProperty("canal.manager.jdbc.url"); remoteConfigLoader = RemoteConfigLoaderFactory.getRemoteConfigLoader(properties);
if (!StringUtils.isEmpty(jdbcUrl)) { if (remoteConfigLoader != null) {
logger.info("## load remote canal configurations");
// load remote config
String jdbcUsername = properties.getProperty("canal.manager.jdbc.username");
String jdbcPassword = properties.getProperty("canal.manager.jdbc.password");
managerDbConfigMonitor = new ManagerRemoteConfigMonitor(jdbcUrl, jdbcUsername, jdbcPassword);
// 加载远程canal.properties // 加载远程canal.properties
Properties remoteConfig = managerDbConfigMonitor.loadRemoteConfig(); Properties remoteConfig = remoteConfigLoader.loadRemoteConfig();
// 加载remote instance配置 // 加载remote instance配置
managerDbConfigMonitor.loadRemoteInstanceConfigs(); remoteConfigLoader.loadRemoteInstanceConfigs();
if (remoteConfig != null) { if (remoteConfig != null) {
properties = remoteConfig; properties = remoteConfig;
} else { } else {
managerDbConfigMonitor = null; remoteConfigLoader = null;
} }
} else {
logger.info("## load canal configurations");
} }
final CanalStater canalStater = new CanalStater(); final CanalStater canalStater = new CanalStater();
canalStater.start(properties); canalStater.start(properties);
if (managerDbConfigMonitor != null) { if (remoteConfigLoader != null) {
managerDbConfigMonitor.start(new ManagerRemoteConfigMonitor.Listener<Properties>() { remoteConfigLoader.startMonitor(new RemoteCanalConfigMonitor() {
@Override @Override
public void onChange(Properties properties) { public void onChange(Properties properties) {
@@ -81,8 +76,8 @@ public class CanalLauncher {
Thread.sleep(1000); Thread.sleep(1000);
} }
if (managerDbConfigMonitor != null) { if (remoteConfigLoader != null) {
managerDbConfigMonitor.destroy(); remoteConfigLoader.destroy();
} }
} catch (Throwable e) { } catch (Throwable e) {
logger.error("## Something goes wrong when starting up the canal Server:", e); logger.error("## Something goes wrong when starting up the canal Server:", e);

5
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java
View File

@@ -150,7 +150,6 @@ public class CanalStater {
if (!StringUtils.isEmpty(acks)) { if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks); mqProperties.setAcks(acks);
} }
String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY); String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) { if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey); mqProperties.setAliyunAccessKey(aliyunAccessKey);
@@ -159,6 +158,10 @@ public class CanalStater {
if (!StringUtils.isEmpty(aliyunSecretKey)) { if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunSecretKey(aliyunSecretKey); mqProperties.setAliyunSecretKey(aliyunSecretKey);
} }
String transaction = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_TRANSACTION);
if (!StringUtils.isEmpty(transaction)) {
mqProperties.setTransaction(Boolean.valueOf(transaction));
}
return mqProperties; return mqProperties;
} }

47
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/ConfigItem.java Normal file
View File

@@ -0,0 +1,47 @@
package com.alibaba.otter.canal.deployer.monitor.remote;
/**
* 配置对应对象
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public class ConfigItem {
private Long id;
private String name;
private String content;
private long modifiedTime;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(long modifiedTime) {
this.modifiedTime = modifiedTime;
}
}

242
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerRemoteConfigMonitor.java → deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -1,39 +1,41 @@
package com.alibaba.otter.canal.deployer.monitor; package com.alibaba.otter.canal.deployer.monitor.remote;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.sql.*; import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.*; import java.util.*;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.deployer.CanalConstants; import com.alibaba.otter.canal.deployer.CanalConstants;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.collect.MapMaker; import com.google.common.collect.MapMaker;
/** /**
* 远程配置装载监控 * 基于数据库的远程配置装载器
* *
* @author rewerma 2018-12-30 下午05:12:16 * @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.1 * @version 1.0.0
*/ */
public class ManagerRemoteConfigMonitor { public class DbRemoteConfigLoader implements RemoteConfigLoader {
private static final Logger logger = LoggerFactory.getLogger(ManagerRemoteConfigMonitor.class); private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
private Map<String, ConfigItem> remoteInstanceConfigs = new MapMaker().makeMap(); private Map<String, ConfigItem> remoteInstanceConfigs = new MapMaker().makeMap();
private Connection conn; private DruidDataSource dataSource;
private String jdbcUrl;
private String jdbcUsername;
private String jdbcPassword;
private long currentConfigTimestamp = 0; private long currentConfigTimestamp = 0;
@@ -41,29 +43,36 @@ public class ManagerRemoteConfigMonitor {
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2, private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
new NamedThreadFactory("remote-canal-config-scan")); new NamedThreadFactory("remote-canal-config-scan"));
public ManagerRemoteConfigMonitor(String jdbcUrl, String jdbcUsername, String jdbcPassword){ private RemoteInstanceMonitor remoteInstanceMonitor = new RemoteInstanceMonitorImpl();
this.jdbcUrl = jdbcUrl;
this.jdbcUsername = jdbcUsername;
this.jdbcPassword = jdbcPassword;
}
public void setScanIntervalInSecond(long scanIntervalInSecond) { public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
this.scanIntervalInSecond = scanIntervalInSecond; dataSource = new DruidDataSource();
} if (StringUtils.isEmpty(driverName)) {
driverName = "com.mysql.jdbc.Driver";
private Connection getConn() throws Exception { }
if (conn == null || conn.isClosed()) { dataSource.setDriverClassName(driverName);
Class.forName("com.mysql.jdbc.Driver"); dataSource.setUrl(jdbcUrl);
conn = DriverManager.getConnection(jdbcUrl, jdbcUsername, jdbcPassword); dataSource.setUsername(jdbcUsername);
dataSource.setPassword(jdbcPassword);
dataSource.setInitialSize(1);
dataSource.setMinIdle(1);
dataSource.setMaxActive(1);
dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000);
try {
dataSource.init();
} catch (SQLException e) {
throw new RuntimeException(e.getMessage(), e);
} }
return conn;
} }
/** /**
* 加载远程 canal.properties文件 覆盖本地 * 加载远程 canal.properties文件
* *
* @return 远程配置的properties * @return 远程配置的properties
*/ */
@Override
public Properties loadRemoteConfig() { public Properties loadRemoteConfig() {
Properties properties = null; Properties properties = null;
try { try {
@@ -87,15 +96,14 @@ public class ManagerRemoteConfigMonitor {
} }
/** /**
* 加载远程的instance配置 覆盖本地 * 覆盖本地 canal.properties
*
* @param content 远程配置内容文本
*/ */
public void loadRemoteInstanceConfigs() { private void overrideLocalCanalConfig(String content) {
try { try (FileWriter writer = new FileWriter(getConfPath() + "canal.properties")) {
// 加载远程instance配置 writer.write(content);
Map<String, ConfigItem>[] modifiedConfigs = getModifiedInstanceConfigs(); writer.flush();
if (modifiedConfigs != null) {
overrideLocalInstanceConfigs(modifiedConfigs);
}
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
@@ -108,7 +116,9 @@ public class ManagerRemoteConfigMonitor {
*/ */
private ConfigItem getRemoteCanalConfig() { private ConfigItem getRemoteCanalConfig() {
String sql = "select name, content, modified_time from canal_config where id=1"; String sql = "select name, content, modified_time from canal_config where id=1";
try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) { try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) { if (rs.next()) {
ConfigItem configItem = new ConfigItem(); ConfigItem configItem = new ConfigItem();
configItem.setId(1L); configItem.setId(1L);
@@ -124,30 +134,27 @@ public class ManagerRemoteConfigMonitor {
} }
/** /**
* 覆盖本地 canal.properties * 加载远程的instance配置
*
* @param content 远程配置内容文本
*/ */
private void overrideLocalCanalConfig(String content) { @Override
try (FileWriter writer = new FileWriter(getConfPath() + "canal.properties")) { public void loadRemoteInstanceConfigs() {
writer.write(content); try {
writer.flush(); // 加载远程instance配置
loadModifiedInstanceConfigs();
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
/** /**
* 获取远程instance新增、修改、删除配置 * 加载远程instance新增、修改、删除配置
*
* @return Map[0]：新增或修改的instance配置; Map[1]：删除的instance配置
*/ */
@SuppressWarnings("unchecked") private void loadModifiedInstanceConfigs() {
private Map<String, ConfigItem>[] getModifiedInstanceConfigs() {
Map<String, ConfigItem>[] res = new Map[2];
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>(); Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
String sql = "select id, name, modified_time from canal_instance_config"; String sql = "select id, name, modified_time from canal_instance_config";
try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(sql)) { try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) { while (rs.next()) {
ConfigItem configItem = new ConfigItem(); ConfigItem configItem = new ConfigItem();
configItem.setId(rs.getLong("id")); configItem.setId(rs.getLong("id"));
@@ -157,7 +164,6 @@ public class ManagerRemoteConfigMonitor {
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
return null;
} }
if (!remoteConfigStatus.isEmpty()) { if (!remoteConfigStatus.isEmpty()) {
@@ -176,10 +182,11 @@ public class ManagerRemoteConfigMonitor {
} }
} }
if (!changedIds.isEmpty()) { if (!changedIds.isEmpty()) {
Map<String, ConfigItem> changedInstanceConfig = new HashMap<>();
String contentsSql = "select id, name, content, modified_time from canal_instance_config where id in (" String contentsSql = "select id, name, content, modified_time from canal_instance_config where id in ("
+ Joiner.on(",").join(changedIds) + ")"; + Joiner.on(",").join(changedIds) + ")";
try (Statement stmt = getConn().createStatement(); ResultSet rs = stmt.executeQuery(contentsSql)) { try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(contentsSql)) {
while (rs.next()) { while (rs.next()) {
ConfigItem configItemNew = new ConfigItem(); ConfigItem configItemNew = new ConfigItem();
configItemNew.setId(rs.getLong("id")); configItemNew.setId(rs.getLong("id"));
@@ -188,64 +195,20 @@ public class ManagerRemoteConfigMonitor {
configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime()); configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
remoteInstanceConfigs.put(configItemNew.getName(), configItemNew); remoteInstanceConfigs.put(configItemNew.getName(), configItemNew);
changedInstanceConfig.put(configItemNew.getName(), configItemNew); remoteInstanceMonitor.onModify(configItemNew);
} }
res[0] = changedInstanceConfig.isEmpty() ? null : changedInstanceConfig;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
} }
Map<String, ConfigItem> removedInstanceConfig = new HashMap<>();
for (String name : remoteInstanceConfigs.keySet()) { for (String name : remoteInstanceConfigs.keySet()) {
if (!remoteConfigStatus.containsKey(name)) { if (!remoteConfigStatus.containsKey(name)) {
// 删除 // 删除
remoteInstanceConfigs.remove(name); remoteInstanceConfigs.remove(name);
removedInstanceConfig.put(name, null); remoteInstanceMonitor.onDelete(name);
}
}
res[1] = removedInstanceConfig.isEmpty() ? null : removedInstanceConfig;
if (res[0] == null && res[1] == null) {
return null;
} else {
return res;
}
}
/**
* 覆盖本地instance配置
*
* @param modifiedInstanceConfigs 有变更的配置项
*/
private void overrideLocalInstanceConfigs(Map<String, ConfigItem>[] modifiedInstanceConfigs) {
Map<String, ConfigItem> changedInstanceConfigs = modifiedInstanceConfigs[0];
if (changedInstanceConfigs != null) {
for (ConfigItem configItem : changedInstanceConfigs.values()) {
File instanceDir = new File(getConfPath() + configItem.getName());
if (!instanceDir.exists()) {
instanceDir.mkdirs();
}
try (FileWriter writer = new FileWriter(
getConfPath() + configItem.getName() + "/instance.properties")) {
writer.write(configItem.getContent());
writer.flush();
logger.info("## Loaded remote instance config: {}/instance.properties ", configItem.getName());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
Map<String, ConfigItem> removedInstanceConfigs = modifiedInstanceConfigs[1];
if (removedInstanceConfigs != null) {
for (String name : removedInstanceConfigs.keySet()) {
File file = new File(getConfPath() + name + "/");
if (file.exists()) {
deleteDir(file);
logger.info("## Deleted and loaded remote instance config: {} ", name);
}
} }
} }
} }
@@ -273,9 +236,9 @@ public class ManagerRemoteConfigMonitor {
/** /**
* 监听 canal 主配置和 instance 配置变化 * 监听 canal 主配置和 instance 配置变化
* *
* @param listener 监听回调方法 * @param remoteCanalConfigMonitor 监听回调方法
*/ */
public void start(final Listener<Properties> listener) { public void startMonitor(final RemoteCanalConfigMonitor remoteCanalConfigMonitor) {
// 监听canal.properties变化 // 监听canal.properties变化
executor.scheduleWithFixedDelay(new Runnable() { executor.scheduleWithFixedDelay(new Runnable() {
@@ -283,10 +246,10 @@ public class ManagerRemoteConfigMonitor {
try { try {
Properties properties = loadRemoteConfig(); Properties properties = loadRemoteConfig();
if (properties != null) { if (properties != null) {
listener.onChange(properties); remoteCanalConfigMonitor.onChange(properties);
} }
} catch (Throwable e) { } catch (Throwable e) {
logger.error("scan failed", e); logger.error("Scan remote canal config failed", e);
} }
} }
@@ -299,7 +262,7 @@ public class ManagerRemoteConfigMonitor {
try { try {
loadRemoteInstanceConfigs(); loadRemoteInstanceConfigs();
} catch (Throwable e) { } catch (Throwable e) {
logger.error("scan failed", e); logger.error("Scan remote instance config failed", e);
} }
} }
@@ -311,12 +274,10 @@ public class ManagerRemoteConfigMonitor {
*/ */
public void destroy() { public void destroy() {
executor.shutdownNow(); executor.shutdownNow();
if (conn != null) { try {
try { dataSource.close();
conn.close(); } catch (Exception e) {
} catch (SQLException e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(), e);
}
} }
} }
@@ -336,50 +297,37 @@ public class ManagerRemoteConfigMonitor {
} }
/** /**
* 配置对应对象 * 远程xxx/instance.properties配置监听器实现
*/ */
public static class ConfigItem { private class RemoteInstanceMonitorImpl implements RemoteInstanceMonitor {
private Long id; @Override
private String name; public void onAdd(ConfigItem configItem) {
private String content; this.onModify(configItem);
private long modifiedTime;
public Long getId() {
return id;
} }
public void setId(Long id) { @Override
this.id = id; public void onModify(ConfigItem configItem) {
File instanceDir = new File(getConfPath() + configItem.getName());
if (!instanceDir.exists()) {
instanceDir.mkdirs();
}
try (FileWriter writer = new FileWriter(getConfPath() + configItem.getName() + "/instance.properties")) {
writer.write(configItem.getContent());
writer.flush();
logger.info("## Loaded remote instance config: {}/instance.properties ", configItem.getName());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
} }
public String getName() { @Override
return name; public void onDelete(String instanceName) {
File file = new File(getConfPath() + instanceName + "/");
if (file.exists()) {
deleteDir(file);
logger.info("## Deleted and loaded remote instance config: {} ", instanceName);
}
} }
public void setName(String name) {
this.name = name;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(long modifiedTime) {
this.modifiedTime = modifiedTime;
}
}
public interface Listener<T> {
void onChange(T properties);
} }
} }

13
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteCanalConfigMonitor.java Normal file
View File

@@ -0,0 +1,13 @@
package com.alibaba.otter.canal.deployer.monitor.remote;
import java.util.Properties;
/**
* 远程canal.properties配置监听器接口
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public interface RemoteCanalConfigMonitor {
void onChange(Properties properties);
}

36
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoader.java Normal file
View File

@@ -0,0 +1,36 @@
package com.alibaba.otter.canal.deployer.monitor.remote;
import java.util.Properties;
/**
* 远程配置装载器接口
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public interface RemoteConfigLoader {
/**
* 加载远程 canal.properties文件
*
* @return 远程配置的properties
*/
Properties loadRemoteConfig();
/**
* 加载远程的instance配置
*/
void loadRemoteInstanceConfigs();
/**
* 启动监听 canal 主配置和 instance 配置变化
*
* @param remoteCanalConfigMonitor 监听回调方法
*/
void startMonitor(final RemoteCanalConfigMonitor remoteCanalConfigMonitor);
/**
* 销毁
*/
void destroy();
}

35
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteConfigLoaderFactory.java Normal file
View File

@@ -0,0 +1,35 @@
package com.alibaba.otter.canal.deployer.monitor.remote;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 远程配置装载器工厂类
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public class RemoteConfigLoaderFactory {
private static final Logger logger = LoggerFactory.getLogger(RemoteConfigLoaderFactory.class);
public static RemoteConfigLoader getRemoteConfigLoader(Properties localProperties) {
String jdbcUrl = localProperties.getProperty("canal.manager.jdbc.url");
if (!StringUtils.isEmpty(jdbcUrl)) {
logger.info("## load remote canal configurations");
// load remote config
String driverName = localProperties.getProperty("canal.manager.jdbc.driverName");
String jdbcUsername = localProperties.getProperty("canal.manager.jdbc.username");
String jdbcPassword = localProperties.getProperty("canal.manager.jdbc.password");
return new DbRemoteConfigLoader(driverName, jdbcUrl, jdbcUsername, jdbcPassword);
}
// 可扩展其它远程配置加载器
logger.info("## load local canal configurations");
return null;
}
}

31
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitor.java Normal file
View File

@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.deployer.monitor.remote;
/**
* 远程xxx/instance.properties配置监听器接口
*
* @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0
*/
public interface RemoteInstanceMonitor {
/**
* 新增配置事件
*
* @param configItem 配置项
*/
void onAdd(ConfigItem configItem);
/**
* 修改配置事件
*
* @param configItem 配置项
*/
void onModify(ConfigItem configItem);
/**
* 删除配置事件
*
* @param instanceName 实例名
*/
void onDelete(String instanceName);
}

2
deployer/src/main/resources/canal.properties
View File

@@ -115,3 +115,5 @@ canal.mq.canalGetTimeout = 100
canal.mq.flatMessage = true canal.mq.flatMessage = true
canal.mq.compressionType = none canal.mq.compressionType = none
canal.mq.acks = all canal.mq.acks = all
# use transaction for kafka flatMessage batch produce
canal.mq.transaction = false

43
meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
View File

@@ -56,6 +56,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public void stop() { public void stop() {
zkClientx = null; //关闭时置空
super.stop(); super.stop();
} }
@@ -101,6 +102,9 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException { public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
if (zkClientx == null) { //重新加载时可能为空
return new ArrayList<ClientIdentity>();
}
String path = ZookeeperPathUtils.getDestinationPath(destination); String path = ZookeeperPathUtils.getDestinationPath(destination);
List<String> childs = null; List<String> childs = null;
try { try {
@@ -160,20 +164,19 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException { public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName); byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
String batchPath = zkClientx.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, String batchPath = zkClientx
data, .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
true);
String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR); String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
return ZookeeperPathUtils.getBatchMarkId(batchIdString); return ZookeeperPathUtils.getBatchMarkId(batchIdString);
} }
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId) public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
throws CanalMetaManagerException { Long batchId) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), String path = ZookeeperPathUtils
clientIdentity.getClientId(), .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
batchId);
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName); byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
zkClientx.createPersistent(path, data, true); zkClientx.createPersistent(path, data, true);
} }
@@ -204,9 +207,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
PositionRange positionRange = getBatch(clientIdentity, batchId); PositionRange positionRange = getBatch(clientIdentity, batchId);
if (positionRange != null) { if (positionRange != null) {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), String path = ZookeeperPathUtils
clientIdentity.getClientId(), .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
batchId);
zkClientx.delete(path); zkClientx.delete(path);
} }
@@ -214,9 +216,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException { public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), String path = ZookeeperPathUtils
clientIdentity.getClientId(), .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
batchId);
byte[] data = zkClientx.readData(path, true); byte[] data = zkClientx.readData(path, true);
if (data == null) { if (data == null) {
return null; return null;
@@ -227,7 +228,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException { public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> batchChilds = zkClientx.getChildren(path); List<String> batchChilds = zkClientx.getChildren(path);
for (String batchChild : batchChilds) { for (String batchChild : batchChilds) {
@@ -237,7 +239,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public PositionRange getLastestBatch(ClientIdentity clientIdentity) { public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = null; List<String> nodes = null;
try { try {
nodes = zkClientx.getChildren(path); nodes = zkClientx.getChildren(path);
@@ -263,7 +266,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public PositionRange getFirstBatch(ClientIdentity clientIdentity) { public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = null; List<String> nodes = null;
try { try {
nodes = zkClientx.getChildren(path); nodes = zkClientx.getChildren(path);
@@ -289,7 +293,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
} }
public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) { public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = null; List<String> nodes = null;
try { try {
nodes = zkClientx.getChildren(path); nodes = zkClientx.getChildren(path);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -708,7 +708,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
} }
} }
buffer.nextValue(fieldMeta.getColumnName(), i, info.type, info.meta, isBinary); buffer.nextValue(columnBuilder.getName(), i, info.type, info.meta, isBinary);
int javaType = buffer.getJavaType(); int javaType = buffer.getJavaType();
if (buffer.isNull()) { if (buffer.isNull()) {
columnBuilder.setIsNull(true); columnBuilder.setIsNull(true);

9
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -23,6 +23,7 @@ public class MQProperties {
private String acks = "all"; private String acks = "all";
private String aliyunAccessKey = ""; private String aliyunAccessKey = "";
private String aliyunSecretKey = ""; private String aliyunSecretKey = "";
private boolean transaction = false; // 是否开启事务
public static class CanalDestination { public static class CanalDestination {
@@ -201,4 +202,12 @@ public class MQProperties {
public void setMaxRequestSize(int maxRequestSize) { public void setMaxRequestSize(int maxRequestSize) {
this.maxRequestSize = maxRequestSize; this.maxRequestSize = maxRequestSize;
} }
public boolean getTransaction() {
return transaction;
}
public void setTransaction(boolean transaction) {
this.transaction = transaction;
}
} }

142
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.kafka;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.ExecutionException;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.KafkaProducer;
@@ -43,12 +44,16 @@ public class CanalKafkaProducer implements CanalMQProducer {
properties.put("bootstrap.servers", kafkaProperties.getServers()); properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", kafkaProperties.getAcks()); properties.put("acks", kafkaProperties.getAcks());
properties.put("compression.type", kafkaProperties.getCompressionType()); properties.put("compression.type", kafkaProperties.getCompressionType());
properties.put("retries", kafkaProperties.getRetries());
properties.put("batch.size", kafkaProperties.getBatchSize()); properties.put("batch.size", kafkaProperties.getBatchSize());
properties.put("linger.ms", kafkaProperties.getLingerMs()); properties.put("linger.ms", kafkaProperties.getLingerMs());
properties.put("max.request.size", kafkaProperties.getMaxRequestSize()); properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
properties.put("buffer.memory", kafkaProperties.getBufferMemory()); properties.put("buffer.memory", kafkaProperties.getBufferMemory());
properties.put("key.serializer", StringSerializer.class.getName()); properties.put("key.serializer", StringSerializer.class.getName());
if(kafkaProperties.getTransaction()){
properties.put("transactional.id", "canal-transactional-id");
} else {
properties.put("retries", kafkaProperties.getRetries());
}
if (!kafkaProperties.getFlatMessage()) { if (!kafkaProperties.getFlatMessage()) {
properties.put("value.serializer", MessageSerializer.class.getName()); properties.put("value.serializer", MessageSerializer.class.getName());
producer = new KafkaProducer<String, Message>(properties); producer = new KafkaProducer<String, Message>(properties);
@@ -56,8 +61,13 @@ public class CanalKafkaProducer implements CanalMQProducer {
properties.put("value.serializer", StringSerializer.class.getName()); properties.put("value.serializer", StringSerializer.class.getName());
producer2 = new KafkaProducer<String, String>(properties); producer2 = new KafkaProducer<String, String>(properties);
} }
if (kafkaProperties.getTransaction()) {
// producer.initTransactions(); if (!kafkaProperties.getFlatMessage()) {
producer.initTransactions();
} else {
producer2.initTransactions();
}
}
} }
@Override @Override
@@ -79,6 +89,17 @@ public class CanalKafkaProducer implements CanalMQProducer {
@Override @Override
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) { public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
// 开启事务，需要kafka版本支持
Producer producerTmp;
if (!kafkaProperties.getFlatMessage()) {
producerTmp = producer;
} else {
producerTmp = producer2;
}
if (kafkaProperties.getTransaction()) {
producerTmp.beginTransaction();
}
try { try {
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) { if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
// 动态topic // 动态topic
@@ -96,78 +117,64 @@ public class CanalKafkaProducer implements CanalMQProducer {
} else { } else {
send(canalDestination, canalDestination.getTopic(), message); send(canalDestination, canalDestination.getTopic(), message);
} }
if (kafkaProperties.getTransaction()) {
producerTmp.commitTransaction();
}
callback.commit(); callback.commit();
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e);
if (kafkaProperties.getTransaction()) {
producerTmp.abortTransaction();
}
callback.rollback(); callback.rollback();
} }
} }
private void send(MQProperties.CanalDestination canalDestination, String topicName, private void send(MQProperties.CanalDestination canalDestination, String topicName,
Message message) throws Exception { Message message) throws Exception {
// producer.beginTransaction();
if (!kafkaProperties.getFlatMessage()) { if (!kafkaProperties.getFlatMessage()) {
try { ProducerRecord<String, Message> record = null;
ProducerRecord<String, Message> record = null; if (canalDestination.getPartition() != null) {
if (canalDestination.getPartition() != null) { record = new ProducerRecord<>(topicName, canalDestination.getPartition(), null, message);
record = new ProducerRecord<>(topicName, canalDestination.getPartition(), null, message); } else {
} else { if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) { Message[] messages = MQMessageUtils.messagePartition(message,
Message[] messages = MQMessageUtils.messagePartition(message, canalDestination.getPartitionsNum(),
canalDestination.getPartitionsNum(), canalDestination.getPartitionHash());
canalDestination.getPartitionHash()); int length = messages.length;
int length = messages.length; for (int i = 0; i < length; i++) {
for (int i = 0; i < length; i++) { Message messagePartition = messages[i];
Message messagePartition = messages[i]; if (messagePartition != null) {
if (messagePartition != null) { record = new ProducerRecord<>(topicName, i, null, messagePartition);
record = new ProducerRecord<>(topicName, i, null, messagePartition);
}
} }
} else {
record = new ProducerRecord<>(topicName, 0, null, message);
} }
} else {
record = new ProducerRecord<>(topicName, 0, null, message);
} }
}
if (record != null) { if (record != null) {
// 同步发送原生message if (kafkaProperties.getTransaction()) {
producer.send(record);
} else {
producer.send(record).get(); producer.send(record).get();
if (logger.isDebugEnabled()) {
logger.debug("Send message to kafka topic: [{}], packet: {}", topicName, message.toString());
}
} }
} catch (Exception e) {
logger.error(e.getMessage(), e); if (logger.isDebugEnabled()) {
// producer.abortTransaction(); logger.debug("Send message to kafka topic: [{}], packet: {}", topicName, message.toString());
throw e; }
} }
} else { } else {
// 发送扁平数据json // 发送扁平数据json
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(message); List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(message);
if (flatMessages != null) { if (flatMessages != null) {
int idx = 0;
int size = flatMessages.size();
for (FlatMessage flatMessage : flatMessages) { for (FlatMessage flatMessage : flatMessages) {
idx++;
if (StringUtils.isEmpty(canalDestination.getPartitionHash())) { if (StringUtils.isEmpty(canalDestination.getPartitionHash())) {
try { Integer partition = canalDestination.getPartition();
Integer partition = canalDestination.getPartition(); if (partition == null) {
if (partition == null) { partition = 0;
partition = 0;
}
ProducerRecord<String, String> record = new ProducerRecord<String, String>(topicName,
partition,
null,
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
if (idx != size) {
producer2.send(record);
} else {
producer2.send(record).get();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
throw e;
} }
produce(topicName, partition, flatMessage);
} else { } else {
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
canalDestination.getPartitionsNum(), canalDestination.getPartitionsNum(),
@@ -176,25 +183,11 @@ public class CanalKafkaProducer implements CanalMQProducer {
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
FlatMessage flatMessagePart = partitionFlatMessage[i]; FlatMessage flatMessagePart = partitionFlatMessage[i];
if (flatMessagePart != null) { if (flatMessagePart != null) {
try { produce(topicName, i, flatMessagePart);
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
topicName,
i,
null,
JSON.toJSONString(flatMessagePart, SerializerFeature.WriteMapNullValue));
if (idx != size) {
producer2.send(record);
} else {
producer2.send(record).get();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
// producer.abortTransaction();
throw e;
}
} }
} }
} }
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("Send flat message to kafka topic: [{}], packet: {}", logger.debug("Send flat message to kafka topic: [{}], packet: {}",
topicName, topicName,
@@ -203,8 +196,19 @@ public class CanalKafkaProducer implements CanalMQProducer {
} }
} }
} }
}
// producer.commitTransaction(); private void produce(String topicName, int partition, FlatMessage flatMessage) throws ExecutionException,
InterruptedException {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(topicName,
partition,
null,
JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue));
if (kafkaProperties.getTransaction()) {
producer2.send(record);
} else {
producer2.send(record).get();
}
} }
} }