mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fix: ES配置文件监控问题 (#2202)
* fix: ES监控配置文件时如果第一次因为文件错误新增失败后,再次修改不会新增的问题 ES监控配置文件添加的Config缓存没有带Instance导致匹配不到的问题 * fix:ES配置文件监控不生效
This commit is contained in:
@@ -84,6 +84,31 @@ public class ESAdapter implements OuterAdapter {
|
|||||||
for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
|
for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
|
||||||
String configName = entry.getKey();
|
String configName = entry.getKey();
|
||||||
ESSyncConfig config = entry.getValue();
|
ESSyncConfig config = entry.getValue();
|
||||||
|
addSyncConfigToCache(configName, config);
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, String> properties = configuration.getProperties();
|
||||||
|
|
||||||
|
String[] hostArray = configuration.getHosts().split(",");
|
||||||
|
String mode = properties.get("mode");
|
||||||
|
if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
|
||||||
|
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
|
||||||
|
} else {
|
||||||
|
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
|
||||||
|
}
|
||||||
|
|
||||||
|
ESTemplate esTemplate = new ESTemplate(esConnection);
|
||||||
|
esSyncService = new ESSyncService(esTemplate);
|
||||||
|
|
||||||
|
esConfigMonitor = new ESConfigMonitor();
|
||||||
|
esConfigMonitor.init(this, envProperties);
|
||||||
|
} catch (Throwable e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addSyncConfigToCache(String configName, ESSyncConfig config) {
|
||||||
|
Properties envProperties = this.envProperties;
|
||||||
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
|
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
|
||||||
config.getEsMapping().setSchemaItem(schemaItem);
|
config.getEsMapping().setSchemaItem(schemaItem);
|
||||||
|
|
||||||
@@ -118,25 +143,6 @@ public class ESAdapter implements OuterAdapter {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, String> properties = configuration.getProperties();
|
|
||||||
|
|
||||||
String[] hostArray = configuration.getHosts().split(",");
|
|
||||||
String mode = properties.get("mode");
|
|
||||||
if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
|
|
||||||
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
|
|
||||||
} else {
|
|
||||||
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
|
|
||||||
}
|
|
||||||
|
|
||||||
ESTemplate esTemplate = new ESTemplate(esConnection);
|
|
||||||
esSyncService = new ESSyncService(esTemplate);
|
|
||||||
|
|
||||||
esConfigMonitor = new ESConfigMonitor();
|
|
||||||
esConfigMonitor.init(this, envProperties);
|
|
||||||
} catch (Throwable e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void sync(List<Dml> dmls) {
|
public void sync(List<Dml> dmls) {
|
||||||
|
|||||||
@@ -106,6 +106,8 @@ public class ESConfigMonitor {
|
|||||||
addConfigToCache(file, config);
|
addConfigToCache(file, config);
|
||||||
|
|
||||||
logger.info("Change a es mapping config: {} of canal adapter", file.getName());
|
logger.info("Change a es mapping config: {} of canal adapter", file.getName());
|
||||||
|
}else{
|
||||||
|
onFileCreate(file);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
@@ -129,27 +131,7 @@ public class ESConfigMonitor {
|
|||||||
|
|
||||||
private void addConfigToCache(File file, ESSyncConfig config) {
|
private void addConfigToCache(File file, ESSyncConfig config) {
|
||||||
esAdapter.getEsSyncConfig().put(file.getName(), config);
|
esAdapter.getEsSyncConfig().put(file.getName(), config);
|
||||||
|
esAdapter.addSyncConfigToCache(file.getName(),config);
|
||||||
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
|
|
||||||
config.getEsMapping().setSchemaItem(schemaItem);
|
|
||||||
|
|
||||||
DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
|
||||||
if (dataSource == null || dataSource.getUrl() == null) {
|
|
||||||
throw new RuntimeException("No data source found: " + config.getDataSourceKey());
|
|
||||||
}
|
|
||||||
Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
|
|
||||||
Matcher matcher = pattern.matcher(dataSource.getUrl());
|
|
||||||
if (!matcher.find()) {
|
|
||||||
throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
|
|
||||||
}
|
|
||||||
String schema = matcher.group(2);
|
|
||||||
|
|
||||||
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
|
|
||||||
Map<String, ESSyncConfig> esSyncConfigMap = esAdapter.getDbTableEsSyncConfig()
|
|
||||||
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>());
|
|
||||||
esSyncConfigMap.put(file.getName(), config);
|
|
||||||
});
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteConfigFromCache(File file) {
|
private void deleteConfigFromCache(File file) {
|
||||||
|
|||||||
Reference in New Issue
Block a user