chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

es配置加载

Browse Source
This commit is contained in:
mcy
2018-10-30 15:57:00 +08:00
parent e4e23c9ecd
commit e1191b665e
7 changed files with 105 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -28,24 +28,17 @@ import com.alibaba.otter.canal.client.adapter.support.SPI;
@SPI("es")
public class ESAdapter implements OuterAdapter {
private static Logger logger = LoggerFactory.getLogger(ESAdapter.class);
private static Logger logger = LoggerFactory.getLogger(ESAdapter.class);
private static volatile Map<String, ESSyncConfig> esSyncConfig = null; // 文件名对应配置
private TransportClient transportClient;
private TransportClient transportClient;
private ESSyncService esSyncService;
private ESSyncService esSyncService;
@Override
public void init(OuterAdapterConfig configuration) {
try {
if (esSyncConfig == null) {
synchronized (ESSyncConfig.class) {
if (esSyncConfig == null) {
esSyncConfig = ESSyncConfigLoader.load();
}
}
}
ESSyncConfigLoader.load();
Map<String, String> properties = configuration.getProperties();
Settings.Builder settingBuilder = Settings.builder();
properties.forEach(settingBuilder::put);

9
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java
View File

@@ -53,6 +53,8 @@ public class ESSyncConfig {
private int commitBatch = 1000;
private String etlCondition;
private SchemaItem schemaItem; // sql解析结果模型
public String get_index() {
return _index;
}
@@ -133,5 +135,12 @@ public class ESSyncConfig {
this.etlCondition = etlCondition;
}
public SchemaItem getSchemaItem() {
return schemaItem;
}
public void setSchemaItem(SchemaItem schemaItem) {
this.schemaItem = schemaItem;
}
}
}

57
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java
View File

@@ -5,27 +5,38 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
public class ESSyncConfigLoader {
private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class);
private static Logger logger = LoggerFactory
.getLogger(ESSyncConfigLoader.class);
private static final String BASE_PATH = "es";
private static final String BASE_PATH = "es";
public static Map<String, ESSyncConfig> load() {
private static volatile Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>(); // 文件名对应配置
private static volatile Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = new LinkedHashMap<>(); // schema-table对应配置
public static Map<String, ESSyncConfig> getEsSyncConfig() {
return esSyncConfig;
}
public static Map<String, List<ESSyncConfig>> getDbTableEsSyncConfig() {
return dbTableEsSyncConfig;
}
public static synchronized void load() {
logger.info("## Start loading mapping config ... ");
Map<String, ESSyncConfig> result = new LinkedHashMap<>();
Collection<String> configs = AdapterConfigs.get("es");
for (String c : configs) {
if (c == null) {
@@ -47,17 +58,35 @@ public class ESSyncConfigLoader {
try {
config.validate();
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
config.getEsMapping().setSchemaItem(schemaItem);
DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
if (dataSource == null || dataSource.getUrl() == null) {
throw new RuntimeException("No data source found: " + config.getDataSourceKey());
}
Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
Matcher matcher = pattern.matcher(dataSource.getUrl());
if (!matcher.find()){
throw new RuntimeException("No found the schema of jdbc-url: " + config.getDataSourceKey());
}
String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
List<ESSyncConfig> esSyncConfigs = dbTableEsSyncConfig
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new ArrayList<>());
esSyncConfigs.add(config);
});
} catch (Exception e) {
throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e);
throw new RuntimeException("ERROR Config: " + c, e);
}
result.put(c, config);
esSyncConfig.put(c, config);
}
logger.info("## Mapping config loaded");
return result;
}
public static String readConfigContent(String config) {
private static String readConfigContent(String config) {
InputStream in = null;
try {
// 先取本地文件，再取类路径
@@ -68,7 +97,7 @@ public class ESSyncConfigLoader {
in = ESSyncConfigLoader.class.getClassLoader().getResourceAsStream(config);
}
if (in == null) {
throw new RuntimeException("Config file not found.");
throw new RuntimeException("Config file: " + config + " not found.");
}
byte[] bytes = new byte[in.available()];

2
client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml
View File

@@ -4,7 +4,7 @@ esMapping:
_type: _doc
_id: id
# pk: id
sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user.a
sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a
left join role b on a.role_id=b.id
left join (select user_id, group_concat(label,',') as labels from user_label
group by user_id) c on c.user_id=a.id"