mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
整理代码
This commit is contained in:
rewer
@@ -1,216 +1,216 @@
|
||||
package com.alibaba.otter.canal.adapter.launcher.loader;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.core.env.EnumerablePropertySource;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.env.PropertySource;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
|
||||
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
|
||||
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||
|
||||
/**
|
||||
* 外部适配器的加载器
|
||||
*
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalAdapterLoader {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
|
||||
|
||||
private CanalClientConfig canalClientConfig;
|
||||
|
||||
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
|
||||
|
||||
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
|
||||
|
||||
private ExtensionLoader<OuterAdapter> loader;
|
||||
|
||||
public CanalAdapterLoader(CanalClientConfig canalClientConfig){
|
||||
this.canalClientConfig = canalClientConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化canal-client
|
||||
*/
|
||||
public void init() {
|
||||
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
|
||||
|
||||
String canalServerHost = this.canalClientConfig.getCanalServerHost();
|
||||
SocketAddress sa = null;
|
||||
if (canalServerHost != null) {
|
||||
String[] ipPort = canalServerHost.split(":");
|
||||
sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
|
||||
}
|
||||
String zkHosts = this.canalClientConfig.getZookeeperHosts();
|
||||
|
||||
if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
|
||||
// 初始化canal-client的适配器
|
||||
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
|
||||
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
|
||||
|
||||
for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
|
||||
List<OuterAdapter> canalOutConnectors = new ArrayList<>();
|
||||
for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
|
||||
loadAdapter(c, canalOutConnectors);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOutConnectors);
|
||||
}
|
||||
CanalAdapterWorker worker;
|
||||
if (sa != null) {
|
||||
worker = new CanalAdapterWorker(canalClientConfig,
|
||||
canalAdapter.getInstance(),
|
||||
sa,
|
||||
canalOuterAdapterGroups);
|
||||
} else if (zkHosts != null) {
|
||||
worker = new CanalAdapterWorker(canalClientConfig,
|
||||
canalAdapter.getInstance(),
|
||||
zkHosts,
|
||||
canalOuterAdapterGroups);
|
||||
} else {
|
||||
throw new RuntimeException("No canal server connector found");
|
||||
}
|
||||
canalWorkers.put(canalAdapter.getInstance(), worker);
|
||||
worker.start();
|
||||
logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
|
||||
}
|
||||
} else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
|
||||
// 初始化canal-client-kafka的适配器
|
||||
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
|
||||
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
|
||||
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
|
||||
List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
|
||||
for (OuterAdapterConfig config : group.getOuterAdapters()) {
|
||||
loadAdapter(config, canalOuterAdapters);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOuterAdapters);
|
||||
|
||||
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
|
||||
canalClientConfig.getMqServers(),
|
||||
canalAdapter.getInstance(),
|
||||
group.getGroupId(),
|
||||
canalOuterAdapterGroups,
|
||||
canalClientConfig.getFlatMessage());
|
||||
canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
|
||||
canalKafkaWorker.start();
|
||||
logger.info("Start adapter for canal-client mq topic: {} succeed",
|
||||
canalAdapter.getInstance() + "-" + group.getGroupId());
|
||||
}
|
||||
}
|
||||
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
|
||||
// 初始化canal-client-rocketMQ的适配器
|
||||
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
|
||||
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
|
||||
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
|
||||
List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
|
||||
for (OuterAdapterConfig config : group.getOuterAdapters()) {
|
||||
loadAdapter(config, canalOuterAdapters);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOuterAdapters);
|
||||
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
|
||||
canalClientConfig.getMqServers(),
|
||||
canalAdapter.getInstance(),
|
||||
group.getGroupId(),
|
||||
canalOuterAdapterGroups,
|
||||
canalClientConfig.getAccessKey(),
|
||||
canalClientConfig.getSecretKey(),
|
||||
canalClientConfig.getFlatMessage());
|
||||
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
|
||||
rocketMQWorker.start();
|
||||
|
||||
logger.info("Start adapter for canal-client mq topic: {} succeed",
|
||||
canalAdapter.getInstance() + "-" + group.getGroupId());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
|
||||
try {
|
||||
OuterAdapter adapter;
|
||||
adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
|
||||
|
||||
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
||||
// 替换ClassLoader
|
||||
Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
|
||||
Environment env = (Environment) SpringContext.getBean(Environment.class);
|
||||
Properties evnProperties = null;
|
||||
if (env instanceof StandardEnvironment) {
|
||||
evnProperties = new Properties();
|
||||
for (PropertySource<?> propertySource : ((StandardEnvironment) env).getPropertySources()) {
|
||||
if (propertySource instanceof EnumerablePropertySource) {
|
||||
String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames();
|
||||
for (String name : names) {
|
||||
Object val = propertySource.getProperty(name);
|
||||
if (val != null) {
|
||||
evnProperties.put(name, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
adapter.init(config, evnProperties);
|
||||
Thread.currentThread().setContextClassLoader(cl);
|
||||
canalOutConnectors.add(adapter);
|
||||
logger.info("Load canal adapter: {} succeed", config.getName());
|
||||
} catch (Exception e) {
|
||||
logger.error("Load canal adapter: {} failed", config.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
|
||||
*/
|
||||
public void destroy() {
|
||||
if (!canalWorkers.isEmpty()) {
|
||||
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
|
||||
List<Future<Boolean>> futures = new ArrayList<>();
|
||||
for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
|
||||
futures.add(stopExecutorService.submit(() -> {
|
||||
canalAdapterWorker.stop();
|
||||
return true;
|
||||
}));
|
||||
}
|
||||
futures.forEach(future -> {
|
||||
try {
|
||||
future.get();
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
stopExecutorService.shutdown();
|
||||
}
|
||||
|
||||
if (!canalMQWorker.isEmpty()) {
|
||||
ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
|
||||
List<Future<Boolean>> futures = new ArrayList<>();
|
||||
for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
|
||||
futures.add(stopMQWorkerService.submit(() -> {
|
||||
canalAdapterMQWorker.stop();
|
||||
return true;
|
||||
}));
|
||||
}
|
||||
futures.forEach(future -> {
|
||||
try {
|
||||
future.get();
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
stopMQWorkerService.shutdown();
|
||||
}
|
||||
logger.info("All canal adapters destroyed");
|
||||
}
|
||||
}
|
||||
package com.alibaba.otter.canal.adapter.launcher.loader;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.core.env.EnumerablePropertySource;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.core.env.PropertySource;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
|
||||
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
|
||||
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||
|
||||
/**
|
||||
* 外部适配器的加载器
|
||||
*
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalAdapterLoader {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
|
||||
|
||||
private CanalClientConfig canalClientConfig;
|
||||
|
||||
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
|
||||
|
||||
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
|
||||
|
||||
private ExtensionLoader<OuterAdapter> loader;
|
||||
|
||||
public CanalAdapterLoader(CanalClientConfig canalClientConfig){
|
||||
this.canalClientConfig = canalClientConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化canal-client
|
||||
*/
|
||||
public void init() {
|
||||
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
|
||||
|
||||
String canalServerHost = this.canalClientConfig.getCanalServerHost();
|
||||
SocketAddress sa = null;
|
||||
if (canalServerHost != null) {
|
||||
String[] ipPort = canalServerHost.split(":");
|
||||
sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
|
||||
}
|
||||
String zkHosts = this.canalClientConfig.getZookeeperHosts();
|
||||
|
||||
if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
|
||||
// 初始化canal-client的适配器
|
||||
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
|
||||
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
|
||||
|
||||
for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
|
||||
List<OuterAdapter> canalOutConnectors = new ArrayList<>();
|
||||
for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
|
||||
loadAdapter(c, canalOutConnectors);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOutConnectors);
|
||||
}
|
||||
CanalAdapterWorker worker;
|
||||
if (sa != null) {
|
||||
worker = new CanalAdapterWorker(canalClientConfig,
|
||||
canalAdapter.getInstance(),
|
||||
sa,
|
||||
canalOuterAdapterGroups);
|
||||
} else if (zkHosts != null) {
|
||||
worker = new CanalAdapterWorker(canalClientConfig,
|
||||
canalAdapter.getInstance(),
|
||||
zkHosts,
|
||||
canalOuterAdapterGroups);
|
||||
} else {
|
||||
throw new RuntimeException("No canal server connector found");
|
||||
}
|
||||
canalWorkers.put(canalAdapter.getInstance(), worker);
|
||||
worker.start();
|
||||
logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
|
||||
}
|
||||
} else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
|
||||
// 初始化canal-client-kafka的适配器
|
||||
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
|
||||
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
|
||||
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
|
||||
List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
|
||||
for (OuterAdapterConfig config : group.getOuterAdapters()) {
|
||||
loadAdapter(config, canalOuterAdapters);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOuterAdapters);
|
||||
|
||||
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
|
||||
canalClientConfig.getMqServers(),
|
||||
canalAdapter.getInstance(),
|
||||
group.getGroupId(),
|
||||
canalOuterAdapterGroups,
|
||||
canalClientConfig.getFlatMessage());
|
||||
canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
|
||||
canalKafkaWorker.start();
|
||||
logger.info("Start adapter for canal-client mq topic: {} succeed",
|
||||
canalAdapter.getInstance() + "-" + group.getGroupId());
|
||||
}
|
||||
}
|
||||
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
|
||||
// 初始化canal-client-rocketMQ的适配器
|
||||
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
|
||||
for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
|
||||
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
|
||||
List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
|
||||
for (OuterAdapterConfig config : group.getOuterAdapters()) {
|
||||
loadAdapter(config, canalOuterAdapters);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOuterAdapters);
|
||||
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
|
||||
canalClientConfig.getMqServers(),
|
||||
canalAdapter.getInstance(),
|
||||
group.getGroupId(),
|
||||
canalOuterAdapterGroups,
|
||||
canalClientConfig.getAccessKey(),
|
||||
canalClientConfig.getSecretKey(),
|
||||
canalClientConfig.getFlatMessage());
|
||||
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
|
||||
rocketMQWorker.start();
|
||||
|
||||
logger.info("Start adapter for canal-client mq topic: {} succeed",
|
||||
canalAdapter.getInstance() + "-" + group.getGroupId());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
|
||||
try {
|
||||
OuterAdapter adapter;
|
||||
adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
|
||||
|
||||
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
||||
// 替换ClassLoader
|
||||
Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
|
||||
Environment env = (Environment) SpringContext.getBean(Environment.class);
|
||||
Properties evnProperties = null;
|
||||
if (env instanceof StandardEnvironment) {
|
||||
evnProperties = new Properties();
|
||||
for (PropertySource<?> propertySource : ((StandardEnvironment) env).getPropertySources()) {
|
||||
if (propertySource instanceof EnumerablePropertySource) {
|
||||
String[] names = ((EnumerablePropertySource<?>) propertySource).getPropertyNames();
|
||||
for (String name : names) {
|
||||
Object val = propertySource.getProperty(name);
|
||||
if (val != null) {
|
||||
evnProperties.put(name, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
adapter.init(config, evnProperties);
|
||||
Thread.currentThread().setContextClassLoader(cl);
|
||||
canalOutConnectors.add(adapter);
|
||||
logger.info("Load canal adapter: {} succeed", config.getName());
|
||||
} catch (Exception e) {
|
||||
logger.error("Load canal adapter: {} failed", config.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
|
||||
*/
|
||||
public void destroy() {
|
||||
if (!canalWorkers.isEmpty()) {
|
||||
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
|
||||
List<Future<Boolean>> futures = new ArrayList<>();
|
||||
for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
|
||||
futures.add(stopExecutorService.submit(() -> {
|
||||
canalAdapterWorker.stop();
|
||||
return true;
|
||||
}));
|
||||
}
|
||||
futures.forEach(future -> {
|
||||
try {
|
||||
future.get();
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
stopExecutorService.shutdown();
|
||||
}
|
||||
|
||||
if (!canalMQWorker.isEmpty()) {
|
||||
ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
|
||||
List<Future<Boolean>> futures = new ArrayList<>();
|
||||
for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
|
||||
futures.add(stopMQWorkerService.submit(() -> {
|
||||
canalAdapterMQWorker.stop();
|
||||
return true;
|
||||
}));
|
||||
}
|
||||
futures.forEach(future -> {
|
||||
try {
|
||||
future.get();
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
stopMQWorkerService.shutdown();
|
||||
}
|
||||
logger.info("All canal adapters destroyed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,248 +1,246 @@
|
||||
package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
|
||||
|
||||
import java.io.FileWriter;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import com.alibaba.otter.canal.common.utils.CommonUtils;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.MapMaker;
|
||||
|
||||
/**
|
||||
* 基于数据库的远程配置装载器
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class DbRemoteConfigLoader implements RemoteConfigLoader {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
|
||||
|
||||
private DruidDataSource dataSource;
|
||||
|
||||
private static volatile long currentConfigTimestamp = 0;
|
||||
private Map<String, ConfigItem> remoteAdapterConfigs = new MapMaker().makeMap();
|
||||
|
||||
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
|
||||
new NamedThreadFactory("remote-adapter-config-scan"));
|
||||
|
||||
private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl();
|
||||
|
||||
public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
|
||||
dataSource = new DruidDataSource();
|
||||
if (StringUtils.isEmpty(driverName)) {
|
||||
driverName = "com.mysql.jdbc.Driver";
|
||||
}
|
||||
dataSource.setDriverClassName(driverName);
|
||||
dataSource.setUrl(jdbcUrl);
|
||||
dataSource.setUsername(jdbcUsername);
|
||||
dataSource.setPassword(jdbcPassword);
|
||||
dataSource.setInitialSize(1);
|
||||
dataSource.setMinIdle(1);
|
||||
dataSource.setMaxActive(1);
|
||||
dataSource.setMaxWait(60000);
|
||||
dataSource.setTimeBetweenEvictionRunsMillis(60000);
|
||||
dataSource.setMinEvictableIdleTimeMillis(300000);
|
||||
try {
|
||||
dataSource.init();
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程application.yml配置
|
||||
*/
|
||||
@Override
|
||||
public void loadRemoteConfig() {
|
||||
try {
|
||||
// 加载远程adapter配置
|
||||
ConfigItem configItem = getRemoteAdapterConfig();
|
||||
if (configItem != null) {
|
||||
if (configItem.getModifiedTime() != currentConfigTimestamp) {
|
||||
currentConfigTimestamp = configItem.getModifiedTime();
|
||||
overrideLocalCanalConfig(configItem.getContent());
|
||||
logger.info("## Loaded remote adapter config: application.yml");
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取远程application.yml配置
|
||||
*
|
||||
* @return 配置对象
|
||||
*/
|
||||
private ConfigItem getRemoteAdapterConfig() {
|
||||
String sql = "select name, content, modified_time from canal_config where id=2";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
if (rs.next()) {
|
||||
ConfigItem configItem = new ConfigItem();
|
||||
configItem.setId(2L);
|
||||
configItem.setName(rs.getString("name"));
|
||||
configItem.setContent(rs.getString("content"));
|
||||
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
return configItem;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 覆盖本地application.yml文件
|
||||
*
|
||||
* @param content 文件内容
|
||||
*/
|
||||
private void overrideLocalCanalConfig(String content) {
|
||||
try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) {
|
||||
writer.write(content);
|
||||
writer.flush();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载adapter配置
|
||||
*/
|
||||
@Override
|
||||
public void loadRemoteAdapterConfigs() {
|
||||
try {
|
||||
// 加载远程adapter配置
|
||||
loadModifiedAdapterConfigs();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载有变动的adapter配置
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private void loadModifiedAdapterConfigs() {
|
||||
Map<String, ConfigItem>[] res = new Map[2];
|
||||
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
|
||||
String sql = "select id, category, name, modified_time from canal_adapter_config";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
while (rs.next()) {
|
||||
ConfigItem configItem = new ConfigItem();
|
||||
configItem.setId(rs.getLong("id"));
|
||||
configItem.setCategory(rs.getString("category"));
|
||||
configItem.setName(rs.getString("name"));
|
||||
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
if (!remoteConfigStatus.isEmpty()) {
|
||||
List<Long> changedIds = new ArrayList<>();
|
||||
|
||||
for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
|
||||
ConfigItem currentConfig = remoteAdapterConfigs
|
||||
.get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName());
|
||||
if (currentConfig == null) {
|
||||
// 新增
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
} else {
|
||||
// 修改
|
||||
if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!changedIds.isEmpty()) {
|
||||
String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in ("
|
||||
+ Joiner.on(",").join(changedIds) + ")";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(contentsSql)) {
|
||||
while (rs.next()) {
|
||||
ConfigItem configItemNew = new ConfigItem();
|
||||
configItemNew.setId(rs.getLong("id"));
|
||||
configItemNew.setCategory(rs.getString("category"));
|
||||
configItemNew.setName(rs.getString("name"));
|
||||
configItemNew.setContent(rs.getString("content"));
|
||||
configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
|
||||
remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(),
|
||||
configItemNew);
|
||||
remoteAdapterMonitor.onModify(configItemNew);
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (ConfigItem configItem : remoteAdapterConfigs.values()) {
|
||||
if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) {
|
||||
// 删除
|
||||
remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName());
|
||||
remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动监听数据库变化
|
||||
*/
|
||||
@Override
|
||||
public void startMonitor() {
|
||||
// 监听application.yml变化
|
||||
executor.scheduleWithFixedDelay(() -> {
|
||||
try {
|
||||
loadRemoteConfig();
|
||||
} catch (Throwable e) {
|
||||
logger.error("scan remote application.yml failed", e);
|
||||
}
|
||||
}, 10, 3, TimeUnit.SECONDS);
|
||||
|
||||
// 监听adapter变化
|
||||
executor.scheduleWithFixedDelay(() -> {
|
||||
try {
|
||||
loadRemoteAdapterConfigs();
|
||||
} catch (Throwable e) {
|
||||
logger.error("scan remote adapter configs failed", e);
|
||||
}
|
||||
}, 10, 3, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 销毁
|
||||
*/
|
||||
@Override
|
||||
public void destroy() {
|
||||
executor.shutdownNow();
|
||||
try {
|
||||
dataSource.close();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
|
||||
|
||||
import java.io.FileWriter;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import com.alibaba.otter.canal.common.utils.CommonUtils;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.MapMaker;
|
||||
|
||||
/**
|
||||
* 基于数据库的远程配置装载器
|
||||
*
|
||||
* @author rewerma 2019-01-25 下午05:20:16
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class DbRemoteConfigLoader implements RemoteConfigLoader {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
|
||||
|
||||
private DruidDataSource dataSource;
|
||||
|
||||
private static volatile long currentConfigTimestamp = 0;
|
||||
private Map<String, ConfigItem> remoteAdapterConfigs = new MapMaker().makeMap();
|
||||
|
||||
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
|
||||
new NamedThreadFactory("remote-adapter-config-scan"));
|
||||
|
||||
private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl();
|
||||
|
||||
public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
|
||||
dataSource = new DruidDataSource();
|
||||
if (StringUtils.isEmpty(driverName)) {
|
||||
driverName = "com.mysql.jdbc.Driver";
|
||||
}
|
||||
dataSource.setDriverClassName(driverName);
|
||||
dataSource.setUrl(jdbcUrl);
|
||||
dataSource.setUsername(jdbcUsername);
|
||||
dataSource.setPassword(jdbcPassword);
|
||||
dataSource.setInitialSize(1);
|
||||
dataSource.setMinIdle(1);
|
||||
dataSource.setMaxActive(1);
|
||||
dataSource.setMaxWait(60000);
|
||||
dataSource.setTimeBetweenEvictionRunsMillis(60000);
|
||||
dataSource.setMinEvictableIdleTimeMillis(300000);
|
||||
try {
|
||||
dataSource.init();
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载远程application.yml配置
|
||||
*/
|
||||
@Override
|
||||
public void loadRemoteConfig() {
|
||||
try {
|
||||
// 加载远程adapter配置
|
||||
ConfigItem configItem = getRemoteAdapterConfig();
|
||||
if (configItem != null) {
|
||||
if (configItem.getModifiedTime() != currentConfigTimestamp) {
|
||||
currentConfigTimestamp = configItem.getModifiedTime();
|
||||
overrideLocalCanalConfig(configItem.getContent());
|
||||
logger.info("## Loaded remote adapter config: application.yml");
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取远程application.yml配置
|
||||
*
|
||||
* @return 配置对象
|
||||
*/
|
||||
private ConfigItem getRemoteAdapterConfig() {
|
||||
String sql = "select name, content, modified_time from canal_config where id=2";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
if (rs.next()) {
|
||||
ConfigItem configItem = new ConfigItem();
|
||||
configItem.setId(2L);
|
||||
configItem.setName(rs.getString("name"));
|
||||
configItem.setContent(rs.getString("content"));
|
||||
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
return configItem;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 覆盖本地application.yml文件
|
||||
*
|
||||
* @param content 文件内容
|
||||
*/
|
||||
private void overrideLocalCanalConfig(String content) {
|
||||
try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) {
|
||||
writer.write(content);
|
||||
writer.flush();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载adapter配置
|
||||
*/
|
||||
@Override
|
||||
public void loadRemoteAdapterConfigs() {
|
||||
try {
|
||||
// 加载远程adapter配置
|
||||
loadModifiedAdapterConfigs();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载有变动的adapter配置
|
||||
*/
|
||||
private void loadModifiedAdapterConfigs() {
|
||||
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
|
||||
String sql = "select id, category, name, modified_time from canal_adapter_config";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
while (rs.next()) {
|
||||
ConfigItem configItem = new ConfigItem();
|
||||
configItem.setId(rs.getLong("id"));
|
||||
configItem.setCategory(rs.getString("category"));
|
||||
configItem.setName(rs.getString("name"));
|
||||
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
if (!remoteConfigStatus.isEmpty()) {
|
||||
List<Long> changedIds = new ArrayList<>();
|
||||
|
||||
for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
|
||||
ConfigItem currentConfig = remoteAdapterConfigs
|
||||
.get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName());
|
||||
if (currentConfig == null) {
|
||||
// 新增
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
} else {
|
||||
// 修改
|
||||
if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
|
||||
changedIds.add(remoteConfigStat.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!changedIds.isEmpty()) {
|
||||
String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in ("
|
||||
+ Joiner.on(",").join(changedIds) + ")";
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement();
|
||||
ResultSet rs = stmt.executeQuery(contentsSql)) {
|
||||
while (rs.next()) {
|
||||
ConfigItem configItemNew = new ConfigItem();
|
||||
configItemNew.setId(rs.getLong("id"));
|
||||
configItemNew.setCategory(rs.getString("category"));
|
||||
configItemNew.setName(rs.getString("name"));
|
||||
configItemNew.setContent(rs.getString("content"));
|
||||
configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
|
||||
|
||||
remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(),
|
||||
configItemNew);
|
||||
remoteAdapterMonitor.onModify(configItemNew);
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (ConfigItem configItem : remoteAdapterConfigs.values()) {
|
||||
if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) {
|
||||
// 删除
|
||||
remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName());
|
||||
remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动监听数据库变化
|
||||
*/
|
||||
@Override
|
||||
public void startMonitor() {
|
||||
// 监听application.yml变化
|
||||
executor.scheduleWithFixedDelay(() -> {
|
||||
try {
|
||||
loadRemoteConfig();
|
||||
} catch (Throwable e) {
|
||||
logger.error("scan remote application.yml failed", e);
|
||||
}
|
||||
}, 10, 3, TimeUnit.SECONDS);
|
||||
|
||||
// 监听adapter变化
|
||||
executor.scheduleWithFixedDelay(() -> {
|
||||
try {
|
||||
loadRemoteAdapterConfigs();
|
||||
} catch (Throwable e) {
|
||||
logger.error("scan remote adapter configs failed", e);
|
||||
}
|
||||
}, 10, 3, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 销毁
|
||||
*/
|
||||
@Override
|
||||
public void destroy() {
|
||||
executor.shutdownNow();
|
||||
try {
|
||||
dataSource.close();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user