chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: ES配置文件监控问题 (#2202)

Browse Source 
* fix:
ES监控配置文件时如果第一次因为文件错误新增失败后，再次修改不会新增的问题
ES监控配置文件添加的Config缓存没有带Instance导致匹配不到的问题

* fix：ES配置文件监控不生效
This commit is contained in:
gaecfov
2019-09-17 09:22:55 +08:00
committed by rewerma
parent 79552e0a37
commit 61726c7e05
2 changed files with 41 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -84,6 +84,31 @@ public class ESAdapter implements OuterAdapter {
for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
String configName = entry.getKey();
ESSyncConfig config = entry.getValue();
addSyncConfigToCache(configName, config);
}
Map<String, String> properties = configuration.getProperties();
String[] hostArray = configuration.getHosts().split(",");
String mode = properties.get("mode");
if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
} else {
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
}
ESTemplate esTemplate = new ESTemplate(esConnection);
esSyncService = new ESSyncService(esTemplate);
esConfigMonitor = new ESConfigMonitor();
esConfigMonitor.init(this, envProperties);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
public void addSyncConfigToCache(String configName, ESSyncConfig config) {
Properties envProperties = this.envProperties;
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
config.getEsMapping().setSchemaItem(schemaItem);
@@ -118,25 +143,6 @@ public class ESAdapter implements OuterAdapter {
});
}
Map<String, String> properties = configuration.getProperties();
String[] hostArray = configuration.getHosts().split(",");
String mode = properties.get("mode");
if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
} else {
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
}
ESTemplate esTemplate = new ESTemplate(esConnection);
esSyncService = new ESSyncService(esTemplate);
esConfigMonitor = new ESConfigMonitor();
esConfigMonitor.init(this, envProperties);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
@Override
public void sync(List<Dml> dmls) {

24
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java
View File

@@ -106,6 +106,8 @@ public class ESConfigMonitor {
addConfigToCache(file, config);
logger.info("Change a es mapping config: {} of canal adapter", file.getName());
}else{
onFileCreate(file);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
@@ -129,27 +131,7 @@ public class ESConfigMonitor {
private void addConfigToCache(File file, ESSyncConfig config) {
esAdapter.getEsSyncConfig().put(file.getName(), config);
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
config.getEsMapping().setSchemaItem(schemaItem);
DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
if (dataSource == null || dataSource.getUrl() == null) {
throw new RuntimeException("No data source found: " + config.getDataSourceKey());
}
Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
Matcher matcher = pattern.matcher(dataSource.getUrl());
if (!matcher.find()) {
throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
}
String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
Map<String, ESSyncConfig> esSyncConfigMap = esAdapter.getDbTableEsSyncConfig()
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>());
esSyncConfigMap.put(file.getName(), config);
});
esAdapter.addSyncConfigToCache(file.getName(),config);
}
private void deleteConfigFromCache(File file) {