chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

adapter 配置修改动态加载

Browse Source
This commit is contained in:
mcy
2018-11-16 18:07:32 +08:00
parent 9dc4d78b3b
commit d34bee3359
15 changed files with 581 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -28,22 +28,31 @@ import com.alibaba.otter.canal.client.adapter.support.*;
@SPI("rdb")
public class RdbAdapter implements OuterAdapter {
private static Logger logger = LoggerFactory.getLogger(RdbAdapter.class);
private static Logger logger = LoggerFactory.getLogger(RdbAdapter.class);
private Map<String, MappingConfig> rdbMapping = new HashMap<>(); // 文件名对应配置
private Map<String, MappingConfig> mappingConfigCache = new HashMap<>(); // 库名-表名对应配置
private Map<String, MappingConfig> rdbMapping = new HashMap<>(); // 文件名对应配置
private Map<String, Map<String, MappingConfig>> mappingConfigCache = new HashMap<>(); // 库名-表名对应配置
private DruidDataSource dataSource;
private DruidDataSource dataSource;
private RdbSyncService rdbSyncService;
private RdbSyncService rdbSyncService;
private int commitSize = 3000;
private int commitSize = 3000;
private volatile boolean running = false;
private volatile boolean running = false;
private List<SimpleDml> dmlList = Collections.synchronizedList(new ArrayList<>());
private Lock syncLock = new ReentrantLock();
private ExecutorService executor = Executors.newFixedThreadPool(1);
private List<SimpleDml> dmlList = Collections
.synchronizedList(new ArrayList<>());
private Lock syncLock = new ReentrantLock();
private ExecutorService executor = Executors.newFixedThreadPool(1);
public Map<String, MappingConfig> getRdbMapping() {
return rdbMapping;
}
public Map<String, Map<String, MappingConfig>> getMappingConfigCache() {
return mappingConfigCache;
}
@Override
public void init(OuterAdapterConfig configuration) {
@@ -56,11 +65,15 @@ public class RdbAdapter implements OuterAdapter {
rdbMapping.put(key, mappingConfig);
}
});
for (MappingConfig mappingConfig : rdbMapping.values()) {
mappingConfigCache
.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+ mappingConfig.getDbMapping().getDatabase() + "." + mappingConfig.getDbMapping().getTable(),
mappingConfig);
for (Map.Entry<String, MappingConfig> entry : rdbMapping.entrySet()) {
String configName = entry.getKey();
MappingConfig mappingConfig = entry.getValue();
Map<String, MappingConfig> configMap = mappingConfigCache
.computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+ mappingConfig.getDbMapping().getDatabase() + "."
+ mappingConfig.getDbMapping().getTable(),
k1 -> new HashMap<>());
configMap.put(configName, mappingConfig);
}
Map<String, String> properties = configuration.getProperties();
@@ -113,16 +126,19 @@ public class RdbAdapter implements OuterAdapter {
String destination = StringUtils.trimToEmpty(dml.getDestination());
String database = dml.getDatabase();
String table = dml.getTable();
MappingConfig config = mappingConfigCache.get(destination + "." + database + "." + table);
Map<String, MappingConfig> configMap = mappingConfigCache.get(destination + "." + database + "." + table);
List<SimpleDml> simpleDmlList = SimpleDml.dml2SimpleDml(dml, config);
if (configMap != null) {
configMap.values().forEach(config -> {
List<SimpleDml> simpleDmlList = SimpleDml.dml2SimpleDml(dml, config);
dmlList.addAll(simpleDmlList);
dmlList.addAll(simpleDmlList);
if (dmlList.size() > commitSize) {
sync();
if (dmlList.size() > commitSize) {
sync();
}
});
}
if (logger.isDebugEnabled()) {
logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
}

141
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java Normal file
View File

@@ -0,0 +1,141 @@
package com.alibaba.otter.canal.client.adapter.rdb.monitor;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;
import org.apache.commons.io.monitor.FileAlterationObserver;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.Util;
public class RdbConfigMonitor {
private static final Logger logger = LoggerFactory.getLogger(RdbConfigMonitor.class);
private static final String adapterName = "rdb";
private String key;
private RdbAdapter rdbAdapter;
private FileAlterationMonitor fileMonitor;
public void init(String key, RdbAdapter rdbAdapter) {
this.key = key;
this.rdbAdapter = rdbAdapter;
File confDir = Util.getConfDirPath(adapterName);
try {
FileAlterationObserver observer = new FileAlterationObserver(confDir,
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
FileListener listener = new FileListener();
observer.addListener(listener);
fileMonitor = new FileAlterationMonitor(3000, observer);
fileMonitor.start();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
public void destroy() {
try {
fileMonitor.stop();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private class FileListener extends FileAlterationListenerAdaptor {
@Override
public void onFileCreate(File file) {
super.onFileCreate(file);
try {
// 加载新增的配置文件
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
config.validate();
if ((key == null && config.getOuterAdapterKey() == null)
|| (key != null && key.equals(config.getOuterAdapterKey()))) {
addConfigToCache(file, config);
logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Override
public void onFileChange(File file) {
super.onFileChange(file);
try {
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
// 加载配置文件
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
config.validate();
if ((key == null && config.getOuterAdapterKey() == null)
|| (key != null && key.equals(config.getOuterAdapterKey()))) {
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
deleteConfigFromCache(file);
}
addConfigToCache(file, config);
} else {
// 不能修改outerAdapterKey
throw new RuntimeException("Outer adapter key not allowed modify");
}
logger.info("Change a rdb mapping config: {} of canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Override
public void onFileDelete(File file) {
super.onFileDelete(file);
try {
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
deleteConfigFromCache(file);
logger.info("Delete a rdb mapping config: {} of canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void addConfigToCache(File file, MappingConfig config) {
rdbAdapter.getRdbMapping().put(file.getName(), config);
Map<String, MappingConfig> configMap = rdbAdapter.getMappingConfigCache()
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "."
+ config.getDbMapping().getDatabase() + "." + config.getDbMapping().getTable(),
k1 -> new HashMap<>());
configMap.put(file.getName(), config);
}
private void deleteConfigFromCache(File file) {
rdbAdapter.getRdbMapping().remove(file.getName());
for (Map<String, MappingConfig> configMap : rdbAdapter.getMappingConfigCache().values()) {
if (configMap != null) {
configMap.remove(file.getName());
}
}
}
}
}