chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

repair issue #3401 (#3923)

Browse Source 
* 解决OutAdapter单例导致的问题，如 1.多个es7 outAdapter的多线程问题（共享同一个es bulk request），2.多个es7 outAdapter共用一份es配置问题

* 1. 优化adapter和监听器之间的代码结构
2. 监听器只处理匹配的配置文件
3. bugfix-es adapter配置文件回调时漏写esVersion
理论上一个adapter实例对应一个config配置文件

Co-authored-by: agapple <jianghang.loujh@alibaba-inc.com>
This commit is contained in:
zhuchao941
2022-05-13 15:02:06 +08:00
committed by GitHub
parent b12945d4a9
commit 4315f203a2
16 changed files with 594 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/PhoenixAdapter.java
View File

@@ -43,7 +43,7 @@ public class PhoenixAdapter implements OuterAdapter {
private Properties envProperties;
private OuterAdapterConfig configuration;
public Map<String, MappingConfig> getPhoenixMapping() {
return phoenixMapping;
@@ -64,14 +64,11 @@ public class PhoenixAdapter implements OuterAdapter {
@Override
public void init(OuterAdapterConfig configuration, Properties envProperties) {
this.envProperties = envProperties;
this.configuration = configuration;
Map<String, MappingConfig> phoenixMappingTmp = ConfigLoader.load(envProperties);
// 过滤不匹配的key的配置
phoenixMappingTmp.forEach((key, mappingConfig) -> {
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|| (mappingConfig.getOuterAdapterKey() != null
&& mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
phoenixMapping.put(key, mappingConfig);
}
phoenixMappingTmp.forEach((key, config) -> {
addConfig(key, config);
});
if (phoenixMapping.isEmpty()) {
@@ -80,24 +77,6 @@ public class PhoenixAdapter implements OuterAdapter {
logger.info("[{}]phoenix config mapping: {}", this, phoenixMapping.keySet());
}
for (Map.Entry<String, MappingConfig> entry : phoenixMapping.entrySet()) {
String configName = entry.getKey();
MappingConfig mappingConfig = entry.getValue();
String key;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+ StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+ mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
} else {
key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+ mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
}
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
k1 -> new ConcurrentHashMap<>());
configMap.put(configName, mappingConfig);
}
Map<String, String> properties = configuration.getProperties();
DriverClass= properties.get("jdbc.driverClassName");
@@ -288,4 +267,60 @@ public class PhoenixAdapter implements OuterAdapter {
phoenixSyncService.close();
}
}
private void addSyncConfigToCache(String configName, MappingConfig mappingConfig) {
String key;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+ StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+ mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
} else {
key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+ mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
}
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
k1 -> new ConcurrentHashMap<>());
configMap.put(configName, mappingConfig);
}
public boolean addConfig(String fileName, MappingConfig config) {
if (match(config)) {
phoenixMapping.put(fileName, config);
addSyncConfigToCache(fileName, config);
FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName,
configuration.getKey());
return true;
}
return false;
}
public void updateConfig(String fileName, MappingConfig config) {
if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey()
.equals(configuration.getKey())) {
// 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
throw new RuntimeException("not allow to change outAdapterKey");
}
phoenixMapping.put(fileName, config);
addSyncConfigToCache(fileName, config);
}
public void deleteConfig(String fileName) {
phoenixMapping.remove(fileName);
for (Map<String, MappingConfig> configMap : mappingConfigCache.values()) {
if (configMap != null) {
configMap.remove(fileName);
}
}
FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
}
private boolean match(MappingConfig config) {
boolean sameMatch = config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
.equalsIgnoreCase(configuration.getKey());
boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
.startsWith(StringUtils
.join(new String[]{Util.AUTO_GENERATED_PREFIX, config.getDestination(),
config.getGroupId()}, '-'));
return sameMatch || prefixMatch;
}
}

60
client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java
View File

@@ -1,23 +1,19 @@
package com.alibaba.otter.canal.client.adapter.phoenix.monitor;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.Util;
import java.io.File;
import java.util.Properties;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;
import org.apache.commons.io.monitor.FileAlterationObserver;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
/**
* phoenix config monitor
*/
@@ -75,11 +71,10 @@ public class PhoenixConfigMonitor {
return;
}
config.validate();
if ((key == null && config.getOuterAdapterKey() == null)
|| (key != null && key.equals(config.getOuterAdapterKey()))) {
addConfigToCache(file, config);
logger.info("Add a new phoenix mapping config: {} to canal adapter", file.getName());
boolean result = phoenixAdapter.addConfig(file.getName(), config);
if (result) {
logger.info("Add a new phoenix mapping config: {} to canal adapter",
file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
@@ -105,16 +100,7 @@ public class PhoenixConfigMonitor {
return;
}
config.validate();
if ((key == null && config.getOuterAdapterKey() == null)
|| (key != null && key.equals(config.getOuterAdapterKey()))) {
if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
deleteConfigFromCache(file);
}
addConfigToCache(file, config);
} else {
// 不能修改outerAdapterKey
throw new RuntimeException("Outer adapter key not allowed modify");
}
phoenixAdapter.updateConfig(file.getName(), config);
logger.info("Change a phoenix mapping config: {} of canal adapter", file.getName());
}
} catch (Exception e) {
@@ -128,40 +114,12 @@ public class PhoenixConfigMonitor {
try {
if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
deleteConfigFromCache(file);
phoenixAdapter.deleteConfig(file.getName());
logger.info("Delete a phoenix mapping config: {} of canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void addConfigToCache(File file, MappingConfig mappingConfig) {
if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
return;
}
phoenixAdapter.getPhoenixMapping().put(file.getName(), mappingConfig);
Map<String, MappingConfig> configMap = phoenixAdapter.getMappingConfigCache()
.computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+ mappingConfig.getDbMapping().getDatabase() + "-"
+ mappingConfig.getDbMapping().getTable().toLowerCase(),
k1 -> new HashMap<>());
configMap.put(file.getName(), mappingConfig);
}
private void deleteConfigFromCache(File file) {
logger.info("deleteConfigFromCache: {}", file.getName());
MappingConfig mappingConfig = phoenixAdapter.getPhoenixMapping().remove(file.getName());
if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
return;
}
for (Map<String, MappingConfig> configMap : phoenixAdapter.getMappingConfigCache().values()) {
if (configMap != null) {
configMap.remove(file.getName());
}
}
}
}
}