chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

es增加relations配置

Browse Source 
配置monitor yml加载使用通用加载类
This commit is contained in:
mcy
2019-03-13 14:34:31 +08:00
parent 43045bc672
commit 05ad671cc1
11 changed files with 241 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -133,7 +133,7 @@ public class ESAdapter implements OuterAdapter {
esSyncService = new ESSyncService(esTemplate);
esConfigMonitor = new ESConfigMonitor();
esConfigMonitor.init(this);
esConfigMonitor.init(this, envProperties);
} catch (Exception e) {
throw new RuntimeException(e);
}

77
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java
View File

@@ -5,6 +5,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
/**
* ES 映射配置
*
@@ -24,6 +26,12 @@ public class ESSyncConfig {
private ESMapping esMapping;
public void validate() {
if (!esMapping.relations.isEmpty()) {
Map<String, RelationMapping> relationMappings = esMapping.relations.get(0);
RelationMapping relationMapping = relationMappings.values().iterator().next();
esMapping.isChild = StringUtils.isNotEmpty(relationMapping.getParent());
}
if (esMapping._index == null) {
throw new NullPointerException("esMapping._index");
}
@@ -31,7 +39,7 @@ public class ESSyncConfig {
throw new NullPointerException("esMapping._type");
}
if (esMapping._id == null && esMapping.getPk() == null) {
throw new NullPointerException("esMapping._id and esMapping.pk");
throw new NullPointerException("esMapping._id or esMapping.pk");
}
if (esMapping.sql == null) {
throw new NullPointerException("esMapping.sql");
@@ -80,23 +88,24 @@ public class ESSyncConfig {
public static class ESMapping {
private String _index;
private String _type;
private String _id;
private boolean upsert = false;
private String pk;
// private String parent;
private String sql;
private String _index;
private String _type;
private String _id;
private boolean upsert = false;
private String pk;
private List<Map<String, RelationMapping>> relations = new ArrayList<>();
private boolean isChild = false;
private String sql;
// 对象字段, 例: objFields:
// - _labels: array:;
private Map<String, String> objFields = new LinkedHashMap<>();
private List<String> skips = new ArrayList<>();
private int commitBatch = 1000;
private String etlCondition;
private boolean syncByTimestamp = false; // 是否按时间戳定时同步
private Long syncInterval; // 同步时间间隔
private Map<String, String> objFields = new LinkedHashMap<>();
private List<String> skips = new ArrayList<>();
private int commitBatch = 1000;
private String etlCondition;
private boolean syncByTimestamp = false; // 是否按时间戳定时同步
private Long syncInterval; // 同步时间间隔
private SchemaItem schemaItem; // sql解析结果模型
private SchemaItem schemaItem; // sql解析结果模型
public String get_index() {
return _index;
@@ -154,6 +163,22 @@ public class ESSyncConfig {
this.skips = skips;
}
public List<Map<String, RelationMapping>> getRelations() {
return relations;
}
public void setRelations(List<Map<String, RelationMapping>> relations) {
this.relations = relations;
}
public boolean isChild() {
return isChild;
}
public void setChild(boolean child) {
isChild = child;
}
public String getSql() {
return sql;
}
@@ -202,4 +227,26 @@ public class ESSyncConfig {
this.schemaItem = schemaItem;
}
}
public static class RelationMapping {
private String name;
private String parent;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getParent() {
return parent;
}
public void setParent(String parent) {
this.parent = parent;
}
}
}

25
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java
View File

@@ -3,9 +3,11 @@ package com.alibaba.otter.canal.client.adapter.es.monitor;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -31,10 +33,13 @@ public class ESConfigMonitor {
private ESAdapter esAdapter;
private Properties envProperties;
private FileAlterationMonitor fileMonitor;
public void init(ESAdapter esAdapter) {
public void init(ESAdapter esAdapter, Properties envProperties) {
this.esAdapter = esAdapter;
this.envProperties = envProperties;
File confDir = Util.getConfDirPath(adapterName);
try {
FileAlterationObserver observer = new FileAlterationObserver(confDir,
@@ -65,11 +70,13 @@ public class ESConfigMonitor {
try {
// 加载新增的配置文件
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class);
config.validate();
addConfigToCache(file, config);
logger.info("Add a new es mapping config: {} to canal adapter", file.getName());
ESSyncConfig config = YmlConfigBinder
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
if (config != null) {
config.validate();
addConfigToCache(file, config);
logger.info("Add a new es mapping config: {} to canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
@@ -88,7 +95,11 @@ public class ESConfigMonitor {
onFileDelete(file);
return;
}
ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class);
ESSyncConfig config = YmlConfigBinder
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
if (config == null) {
return;
}
config.validate();
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
deleteConfigFromCache(file);