chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

修改yml配置读取绑定，遵循spring yml加载方式，支持environment级占位符替换

Browse Source
This commit is contained in:
mcy
2019-02-01 10:11:52 +08:00
parent a20960834b
commit 8211ef35e6
45 changed files with 4195 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
client-adapter/elasticsearch/pom.xml
View File

@@ -18,12 +18,6 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.19</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.fastsql</groupId>
<artifactId>fastsql</artifactId>
@@ -92,4 +86,4 @@
</plugins>
</build>
</project>
</project>

15
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -1,10 +1,7 @@
package com.alibaba.otter.canal.client.adapter.es;
import java.net.InetAddress;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -28,11 +25,7 @@ import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.client.adapter.support.*;
/**
* ES外部适配器
@@ -69,9 +62,9 @@ public class ESAdapter implements OuterAdapter {
}
@Override
public void init(OuterAdapterConfig configuration) {
public void init(OuterAdapterConfig configuration, Properties envProperties) {
try {
Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load();
Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties);
// 过滤不匹配的key的配置
esSyncConfigTmp.forEach((key, config) -> {
if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)

12
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java
View File

@@ -2,12 +2,12 @@ package com.alibaba.otter.canal.client.adapter.es.config;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
/**
@@ -20,17 +20,15 @@ public class ESSyncConfigLoader {
private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class);
@SuppressWarnings("unchecked")
public static synchronized Map<String, ESSyncConfig> load() {
public static synchronized Map<String, ESSyncConfig> load(Properties envProperties) {
logger.info("## Start loading es mapping config ... ");
Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>();
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("es");
configContentMap.forEach((fileName, content) -> {
Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定
JSONObject configJson = new JSONObject(configMap);
ESSyncConfig config = configJson.toJavaObject(ESSyncConfig.class);
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties);
try {
config.validate();
} catch (Exception e) {

2
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java
View File

@@ -21,7 +21,7 @@ public class ConfigLoadTest {
@Test
public void testLoad() {
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load();
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load(null);
ESSyncConfig config = configMap.get("mytest_user.yml");
Assert.assertNotNull(config);
Assert.assertEquals("defaultDS", config.getDataSourceKey());

2
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java
View File

@@ -26,7 +26,7 @@ public class Common {
outerAdapterConfig.setProperties(properties);
ESAdapter esAdapter = new ESAdapter();
esAdapter.init(outerAdapterConfig);
esAdapter.init(outerAdapterConfig, null);
return esAdapter;
}