mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
use es6x/es7x instead of elaticsearch project
This commit is contained in:
agapple
@@ -1,105 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>canal.client-adapter</artifactId>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<version>1.1.5-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>client-adapter.elasticsearch</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal client adapter elasticsearch module for otter ${project.version}</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>client-adapter.common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.fastsql</groupId>
|
||||
<artifactId>fastsql</artifactId>
|
||||
<version>2.0.0_preview_855</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch</groupId>
|
||||
<artifactId>elasticsearch</artifactId>
|
||||
<version>6.4.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
<version>6.4.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-client</artifactId>
|
||||
<version>6.4.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
<version>6.4.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.12</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>5.1.40</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4</version>
|
||||
<configuration>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<tasks>
|
||||
<copy todir="${project.basedir}/../launcher/target/classes/es" overwrite="true">
|
||||
<fileset dir="${project.basedir}/target/classes/es" erroronmissingdir="true">
|
||||
<include name="*.yml" />
|
||||
</fileset>
|
||||
</copy>
|
||||
</tasks>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -1,256 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
|
||||
import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
|
||||
import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
|
||||
import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
|
||||
import com.alibaba.otter.canal.client.adapter.support.*;
|
||||
|
||||
/**
|
||||
* ES外部适配器
|
||||
*
|
||||
* @author rewerma 2018-10-20
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SPI("es")
|
||||
public class ESAdapter implements OuterAdapter {
|
||||
|
||||
private Map<String, ESSyncConfig> esSyncConfig = new ConcurrentHashMap<>(); // 文件名对应配置
|
||||
private Map<String, Map<String, ESSyncConfig>> dbTableEsSyncConfig = new ConcurrentHashMap<>(); // schema-table对应配置
|
||||
|
||||
private ESConnection esConnection;
|
||||
|
||||
private ESSyncService esSyncService;
|
||||
|
||||
private ESConfigMonitor esConfigMonitor;
|
||||
|
||||
private Properties envProperties;
|
||||
|
||||
public ESSyncService getEsSyncService() {
|
||||
return esSyncService;
|
||||
}
|
||||
|
||||
public Map<String, ESSyncConfig> getEsSyncConfig() {
|
||||
return esSyncConfig;
|
||||
}
|
||||
|
||||
public Map<String, Map<String, ESSyncConfig>> getDbTableEsSyncConfig() {
|
||||
return dbTableEsSyncConfig;
|
||||
}
|
||||
|
||||
public ESConnection getEsConnection() {
|
||||
return esConnection;
|
||||
}
|
||||
|
||||
public void setEsConnection(ESConnection esConnection) {
|
||||
this.esConnection = esConnection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(OuterAdapterConfig configuration, Properties envProperties) {
|
||||
try {
|
||||
this.envProperties = envProperties;
|
||||
Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties);
|
||||
// 过滤不匹配的key的配置
|
||||
esSyncConfigTmp.forEach((key, config) -> {
|
||||
if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
|
||||
|| (config.getOuterAdapterKey() != null
|
||||
&& config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
|
||||
esSyncConfig.put(key, config);
|
||||
}
|
||||
});
|
||||
|
||||
for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
|
||||
String configName = entry.getKey();
|
||||
ESSyncConfig config = entry.getValue();
|
||||
addSyncConfigToCache(configName, config);
|
||||
}
|
||||
|
||||
Map<String, String> properties = configuration.getProperties();
|
||||
|
||||
String[] hostArray = configuration.getHosts().split(",");
|
||||
String mode = properties.get("mode");
|
||||
if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
|
||||
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
|
||||
} else {
|
||||
esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
|
||||
}
|
||||
|
||||
ESTemplate esTemplate = new ESTemplate(esConnection);
|
||||
esSyncService = new ESSyncService(esTemplate);
|
||||
|
||||
esConfigMonitor = new ESConfigMonitor();
|
||||
esConfigMonitor.init(this, envProperties);
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void addSyncConfigToCache(String configName, ESSyncConfig config) {
|
||||
Properties envProperties = this.envProperties;
|
||||
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
|
||||
config.getEsMapping().setSchemaItem(schemaItem);
|
||||
|
||||
DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
if (dataSource == null || dataSource.getUrl() == null) {
|
||||
throw new RuntimeException("No data source found: " + config.getDataSourceKey());
|
||||
}
|
||||
Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
|
||||
Matcher matcher = pattern.matcher(dataSource.getUrl());
|
||||
if (!matcher.find()) {
|
||||
throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
|
||||
}
|
||||
String schema = matcher.group(2);
|
||||
|
||||
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
|
||||
Map<String, ESSyncConfig> esSyncConfigMap;
|
||||
if (envProperties != null
|
||||
&& !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
||||
esSyncConfigMap = dbTableEsSyncConfig
|
||||
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
|
||||
+ StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-"
|
||||
+ tableItem.getTableName(),
|
||||
k -> new ConcurrentHashMap<>());
|
||||
} else {
|
||||
esSyncConfigMap = dbTableEsSyncConfig
|
||||
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-"
|
||||
+ tableItem.getTableName(),
|
||||
k -> new ConcurrentHashMap<>());
|
||||
}
|
||||
|
||||
esSyncConfigMap.put(configName, config);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void sync(List<Dml> dmls) {
|
||||
if (dmls == null || dmls.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (Dml dml : dmls) {
|
||||
if (!dml.getIsDdl()) {
|
||||
sync(dml);
|
||||
}
|
||||
}
|
||||
esSyncService.commit(); // 批次统一提交
|
||||
|
||||
}
|
||||
|
||||
private void sync(Dml dml) {
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> configMap;
|
||||
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
||||
configMap = dbTableEsSyncConfig
|
||||
.get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
|
||||
+ "_" + database + "-" + table);
|
||||
} else {
|
||||
configMap = dbTableEsSyncConfig
|
||||
.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
|
||||
}
|
||||
|
||||
if (configMap != null && !configMap.values().isEmpty()) {
|
||||
esSyncService.sync(configMap.values(), dml);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public EtlResult etl(String task, List<String> params) {
|
||||
EtlResult etlResult = new EtlResult();
|
||||
ESSyncConfig config = esSyncConfig.get(task);
|
||||
if (config != null) {
|
||||
DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
ESEtlService esEtlService = new ESEtlService(esConnection, config);
|
||||
if (dataSource != null) {
|
||||
return esEtlService.importData(params);
|
||||
} else {
|
||||
etlResult.setSucceeded(false);
|
||||
etlResult.setErrorMessage("DataSource not found");
|
||||
return etlResult;
|
||||
}
|
||||
} else {
|
||||
StringBuilder resultMsg = new StringBuilder();
|
||||
boolean resSuccess = true;
|
||||
for (ESSyncConfig configTmp : esSyncConfig.values()) {
|
||||
// 取所有的destination为task的配置
|
||||
if (configTmp.getDestination().equals(task)) {
|
||||
ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
|
||||
EtlResult etlRes = esEtlService.importData(params);
|
||||
if (!etlRes.getSucceeded()) {
|
||||
resSuccess = false;
|
||||
resultMsg.append(etlRes.getErrorMessage()).append("\n");
|
||||
} else {
|
||||
resultMsg.append(etlRes.getResultMessage()).append("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (resultMsg.length() > 0) {
|
||||
etlResult.setSucceeded(resSuccess);
|
||||
if (resSuccess) {
|
||||
etlResult.setResultMessage(resultMsg.toString());
|
||||
} else {
|
||||
etlResult.setErrorMessage(resultMsg.toString());
|
||||
}
|
||||
return etlResult;
|
||||
}
|
||||
}
|
||||
etlResult.setSucceeded(false);
|
||||
etlResult.setErrorMessage("Task not found");
|
||||
return etlResult;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> count(String task) {
|
||||
ESSyncConfig config = esSyncConfig.get(task);
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).size(0)
|
||||
.getResponse();
|
||||
|
||||
long rowCount = response.getHits().getTotalHits();
|
||||
Map<String, Object> res = new LinkedHashMap<>();
|
||||
res.put("esIndex", mapping.get_index());
|
||||
res.put("count", rowCount);
|
||||
return res;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (esConfigMonitor != null) {
|
||||
esConfigMonitor.destroy();
|
||||
}
|
||||
if (esConnection != null) {
|
||||
esConnection.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDestination(String task) {
|
||||
ESSyncConfig config = esSyncConfig.get(task);
|
||||
if (config != null) {
|
||||
return config.getDestination();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.config;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* ES 映射配置
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESSyncConfig implements AdapterConfig {
|
||||
|
||||
private String dataSourceKey; // 数据源key
|
||||
|
||||
private String outerAdapterKey; // adapter key
|
||||
|
||||
private String groupId; // group id
|
||||
|
||||
private String destination; // canal destination
|
||||
|
||||
private ESMapping esMapping;
|
||||
|
||||
public void validate() {
|
||||
if (esMapping._index == null) {
|
||||
throw new NullPointerException("esMapping._index");
|
||||
}
|
||||
if (esMapping._type == null) {
|
||||
throw new NullPointerException("esMapping._type");
|
||||
}
|
||||
if (esMapping._id == null && esMapping.getPk() == null) {
|
||||
throw new NullPointerException("esMapping._id or esMapping.pk");
|
||||
}
|
||||
if (esMapping.sql == null) {
|
||||
throw new NullPointerException("esMapping.sql");
|
||||
}
|
||||
}
|
||||
|
||||
public String getDataSourceKey() {
|
||||
return dataSourceKey;
|
||||
}
|
||||
|
||||
public void setDataSourceKey(String dataSourceKey) {
|
||||
this.dataSourceKey = dataSourceKey;
|
||||
}
|
||||
|
||||
public String getOuterAdapterKey() {
|
||||
return outerAdapterKey;
|
||||
}
|
||||
|
||||
public void setOuterAdapterKey(String outerAdapterKey) {
|
||||
this.outerAdapterKey = outerAdapterKey;
|
||||
}
|
||||
|
||||
public String getGroupId() {
|
||||
return groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(String groupId) {
|
||||
this.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getDestination() {
|
||||
return destination;
|
||||
}
|
||||
|
||||
public void setDestination(String destination) {
|
||||
this.destination = destination;
|
||||
}
|
||||
|
||||
public ESMapping getEsMapping() {
|
||||
return esMapping;
|
||||
}
|
||||
|
||||
public void setEsMapping(ESMapping esMapping) {
|
||||
this.esMapping = esMapping;
|
||||
}
|
||||
|
||||
public ESMapping getMapping(){
|
||||
return esMapping;
|
||||
}
|
||||
|
||||
public static class ESMapping implements AdapterMapping{
|
||||
|
||||
private String _index;
|
||||
private String _type;
|
||||
private String _id;
|
||||
private boolean upsert = false;
|
||||
private String pk;
|
||||
private Map<String, RelationMapping> relations = new LinkedHashMap<>();
|
||||
private String sql;
|
||||
// 对象字段, 例: objFields:
|
||||
// - _labels: array:;
|
||||
private Map<String, String> objFields = new LinkedHashMap<>();
|
||||
private List<String> skips = new ArrayList<>();
|
||||
private int commitBatch = 1000;
|
||||
private String etlCondition;
|
||||
private boolean syncByTimestamp = false; // 是否按时间戳定时同步
|
||||
private Long syncInterval; // 同步时间间隔
|
||||
|
||||
private SchemaItem schemaItem; // sql解析结果模型
|
||||
|
||||
public String get_index() {
|
||||
return _index;
|
||||
}
|
||||
|
||||
public void set_index(String _index) {
|
||||
this._index = _index;
|
||||
}
|
||||
|
||||
public String get_type() {
|
||||
return _type;
|
||||
}
|
||||
|
||||
public void set_type(String _type) {
|
||||
this._type = _type;
|
||||
}
|
||||
|
||||
public String get_id() {
|
||||
return _id;
|
||||
}
|
||||
|
||||
public void set_id(String _id) {
|
||||
this._id = _id;
|
||||
}
|
||||
|
||||
public boolean isUpsert() {
|
||||
return upsert;
|
||||
}
|
||||
|
||||
public void setUpsert(boolean upsert) {
|
||||
this.upsert = upsert;
|
||||
}
|
||||
|
||||
public String getPk() {
|
||||
return pk;
|
||||
}
|
||||
|
||||
public void setPk(String pk) {
|
||||
this.pk = pk;
|
||||
}
|
||||
|
||||
public Map<String, String> getObjFields() {
|
||||
return objFields;
|
||||
}
|
||||
|
||||
public void setObjFields(Map<String, String> objFields) {
|
||||
this.objFields = objFields;
|
||||
}
|
||||
|
||||
public List<String> getSkips() {
|
||||
return skips;
|
||||
}
|
||||
|
||||
public void setSkips(List<String> skips) {
|
||||
this.skips = skips;
|
||||
}
|
||||
|
||||
public Map<String, RelationMapping> getRelations() {
|
||||
return relations;
|
||||
}
|
||||
|
||||
public void setRelations(Map<String, RelationMapping> relations) {
|
||||
this.relations = relations;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public int getCommitBatch() {
|
||||
return commitBatch;
|
||||
}
|
||||
|
||||
public void setCommitBatch(int commitBatch) {
|
||||
this.commitBatch = commitBatch;
|
||||
}
|
||||
|
||||
public String getEtlCondition() {
|
||||
return etlCondition;
|
||||
}
|
||||
|
||||
public void setEtlCondition(String etlCondition) {
|
||||
this.etlCondition = etlCondition;
|
||||
}
|
||||
|
||||
public Long getSyncInterval() {
|
||||
return syncInterval;
|
||||
}
|
||||
|
||||
public void setSyncInterval(Long syncInterval) {
|
||||
this.syncInterval = syncInterval;
|
||||
}
|
||||
|
||||
public boolean isSyncByTimestamp() {
|
||||
return syncByTimestamp;
|
||||
}
|
||||
|
||||
public void setSyncByTimestamp(boolean syncByTimestamp) {
|
||||
this.syncByTimestamp = syncByTimestamp;
|
||||
}
|
||||
|
||||
public SchemaItem getSchemaItem() {
|
||||
return schemaItem;
|
||||
}
|
||||
|
||||
public void setSchemaItem(SchemaItem schemaItem) {
|
||||
this.schemaItem = schemaItem;
|
||||
}
|
||||
}
|
||||
|
||||
public static class RelationMapping {
|
||||
|
||||
private String name;
|
||||
private String parent;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getParent() {
|
||||
return parent;
|
||||
}
|
||||
|
||||
public void setParent(String parent) {
|
||||
this.parent = parent;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.config;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
|
||||
/**
|
||||
* ES 配置装载器
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESSyncConfigLoader {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class);
|
||||
|
||||
public static synchronized Map<String, ESSyncConfig> load(Properties envProperties) {
|
||||
logger.info("## Start loading es mapping config ... ");
|
||||
|
||||
Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>();
|
||||
|
||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("es");
|
||||
configContentMap.forEach((fileName, content) -> {
|
||||
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
config.validate();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
|
||||
}
|
||||
esSyncConfig.put(fileName, config);
|
||||
});
|
||||
|
||||
logger.info("## ES mapping config loaded");
|
||||
return esSyncConfig;
|
||||
}
|
||||
}
|
||||
@@ -1,440 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.config;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
|
||||
|
||||
/**
|
||||
* ES 映射配置视图
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class SchemaItem {
|
||||
|
||||
private Map<String, TableItem> aliasTableItems = new LinkedHashMap<>(); // 别名对应表名
|
||||
private Map<String, FieldItem> selectFields = new LinkedHashMap<>(); // 查询字段
|
||||
private String sql;
|
||||
|
||||
private volatile Map<String, List<TableItem>> tableItemAliases;
|
||||
private volatile Map<String, List<FieldItem>> columnFields;
|
||||
private volatile Boolean allFieldsSimple;
|
||||
|
||||
public void init() {
|
||||
this.getTableItemAliases();
|
||||
this.getColumnFields();
|
||||
this.isAllFieldsSimple();
|
||||
aliasTableItems.values().forEach(tableItem -> {
|
||||
tableItem.getRelationTableFields();
|
||||
tableItem.getRelationSelectFieldItems();
|
||||
});
|
||||
}
|
||||
|
||||
public Map<String, TableItem> getAliasTableItems() {
|
||||
return aliasTableItems;
|
||||
}
|
||||
|
||||
public void setAliasTableItems(Map<String, TableItem> aliasTableItems) {
|
||||
this.aliasTableItems = aliasTableItems;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public Map<String, FieldItem> getSelectFields() {
|
||||
return selectFields;
|
||||
}
|
||||
|
||||
public void setSelectFields(Map<String, FieldItem> selectFields) {
|
||||
this.selectFields = selectFields;
|
||||
}
|
||||
|
||||
public String toSql() {
|
||||
// todo
|
||||
return null;
|
||||
}
|
||||
|
||||
public Map<String, List<TableItem>> getTableItemAliases() {
|
||||
if (tableItemAliases == null) {
|
||||
synchronized (SchemaItem.class) {
|
||||
if (tableItemAliases == null) {
|
||||
tableItemAliases = new LinkedHashMap<>();
|
||||
aliasTableItems.forEach((alias, tableItem) -> {
|
||||
List<TableItem> aliases = tableItemAliases
|
||||
.computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>());
|
||||
aliases.add(tableItem);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return tableItemAliases;
|
||||
}
|
||||
|
||||
public Map<String, List<FieldItem>> getColumnFields() {
|
||||
if (columnFields == null) {
|
||||
synchronized (SchemaItem.class) {
|
||||
if (columnFields == null) {
|
||||
columnFields = new LinkedHashMap<>();
|
||||
getSelectFields()
|
||||
.forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> {
|
||||
// TableItem tableItem = getAliasTableItems().get(columnItem.getOwner());
|
||||
// if (!tableItem.isSubQuery()) {
|
||||
//当数据列并非原始列时，columnName是空的，例如concat('px',id)
|
||||
if(columnItem.getColumnName() != null) {
|
||||
List<FieldItem> fieldItems = columnFields.computeIfAbsent(
|
||||
columnItem.getOwner() + "." + columnItem.getColumnName(),
|
||||
k -> new ArrayList<>());
|
||||
fieldItems.add(fieldItem);
|
||||
}
|
||||
// } else {
|
||||
// tableItem.getSubQueryFields().forEach(subQueryField -> {
|
||||
// List<FieldItem> fieldItems = columnFields.computeIfAbsent(
|
||||
// columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(),
|
||||
// k -> new ArrayList<>());
|
||||
// fieldItems.add(fieldItem);
|
||||
// });
|
||||
// }
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
return columnFields;
|
||||
}
|
||||
|
||||
public boolean isAllFieldsSimple() {
|
||||
if (allFieldsSimple == null) {
|
||||
synchronized (SchemaItem.class) {
|
||||
if (allFieldsSimple == null) {
|
||||
allFieldsSimple = true;
|
||||
|
||||
for (FieldItem fieldItem : getSelectFields().values()) {
|
||||
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
|
||||
allFieldsSimple = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allFieldsSimple;
|
||||
}
|
||||
|
||||
public TableItem getMainTable() {
|
||||
if (!aliasTableItems.isEmpty()) {
|
||||
return aliasTableItems.values().iterator().next();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public FieldItem getIdFieldItem(ESMapping mapping) {
|
||||
if (mapping.get_id() != null) {
|
||||
return getSelectFields().get(mapping.get_id());
|
||||
} else {
|
||||
return getSelectFields().get(mapping.getPk());
|
||||
}
|
||||
}
|
||||
|
||||
public static class TableItem {
|
||||
|
||||
private SchemaItem schemaItem;
|
||||
|
||||
private String schema;
|
||||
private String tableName;
|
||||
private String alias;
|
||||
private String subQuerySql;
|
||||
private List<FieldItem> subQueryFields = new ArrayList<>();
|
||||
private List<RelationFieldsPair> relationFields = new ArrayList<>();
|
||||
|
||||
private boolean main;
|
||||
private boolean subQuery;
|
||||
|
||||
private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段对应主表查询字段
|
||||
private volatile List<FieldItem> relationSelectFieldItems; // 子表所在主表的查询字段
|
||||
|
||||
public TableItem(SchemaItem schemaItem){
|
||||
this.schemaItem = schemaItem;
|
||||
}
|
||||
|
||||
public SchemaItem getSchemaItem() {
|
||||
return schemaItem;
|
||||
}
|
||||
|
||||
public void setSchemaItem(SchemaItem schemaItem) {
|
||||
this.schemaItem = schemaItem;
|
||||
}
|
||||
|
||||
public String getSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
public void setSchema(String schema) {
|
||||
this.schema = schema;
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return tableName;
|
||||
}
|
||||
|
||||
public void setTableName(String tableName) {
|
||||
this.tableName = tableName;
|
||||
}
|
||||
|
||||
public String getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
|
||||
public String getSubQuerySql() {
|
||||
return subQuerySql;
|
||||
}
|
||||
|
||||
public void setSubQuerySql(String subQuerySql) {
|
||||
this.subQuerySql = subQuerySql;
|
||||
}
|
||||
|
||||
public boolean isMain() {
|
||||
return main;
|
||||
}
|
||||
|
||||
public void setMain(boolean main) {
|
||||
this.main = main;
|
||||
}
|
||||
|
||||
public boolean isSubQuery() {
|
||||
return subQuery;
|
||||
}
|
||||
|
||||
public void setSubQuery(boolean subQuery) {
|
||||
this.subQuery = subQuery;
|
||||
}
|
||||
|
||||
public List<FieldItem> getSubQueryFields() {
|
||||
return subQueryFields;
|
||||
}
|
||||
|
||||
public void setSubQueryFields(List<FieldItem> subQueryFields) {
|
||||
this.subQueryFields = subQueryFields;
|
||||
}
|
||||
|
||||
public List<RelationFieldsPair> getRelationFields() {
|
||||
return relationFields;
|
||||
}
|
||||
|
||||
public void setRelationFields(List<RelationFieldsPair> relationFields) {
|
||||
this.relationFields = relationFields;
|
||||
}
|
||||
|
||||
public Map<FieldItem, List<FieldItem>> getRelationTableFields() {
|
||||
if (relationTableFields == null) {
|
||||
synchronized (SchemaItem.class) {
|
||||
if (relationTableFields == null) {
|
||||
relationTableFields = new LinkedHashMap<>();
|
||||
|
||||
getRelationFields().forEach(relationFieldsPair -> {
|
||||
FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
|
||||
FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
|
||||
FieldItem currentTableRelField = null;
|
||||
if (getAlias().equals(leftFieldItem.getOwner())) {
|
||||
currentTableRelField = leftFieldItem;
|
||||
} else if (getAlias().equals(rightFieldItem.getOwner())) {
|
||||
currentTableRelField = rightFieldItem;
|
||||
}
|
||||
|
||||
if (currentTableRelField != null) {
|
||||
List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
|
||||
.get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName());
|
||||
if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
|
||||
relationTableFields.put(currentTableRelField, selectFieldItem);
|
||||
} else {
|
||||
selectFieldItem = getSchemaItem().getColumnFields()
|
||||
.get(rightFieldItem.getOwner() + "."
|
||||
+ rightFieldItem.getColumn().getColumnName());
|
||||
if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
|
||||
relationTableFields.put(currentTableRelField, selectFieldItem);
|
||||
} else {
|
||||
throw new UnsupportedOperationException(
|
||||
"Relation condition column must in select columns.");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return relationTableFields;
|
||||
}
|
||||
|
||||
public List<FieldItem> getRelationSelectFieldItems() {
|
||||
if (relationSelectFieldItems == null) {
|
||||
synchronized (SchemaItem.class) {
|
||||
if (relationSelectFieldItems == null) {
|
||||
List<FieldItem> relationSelectFieldItemsTmp = new ArrayList<>();
|
||||
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
if (fieldItem.getOwners().contains(getAlias())) {
|
||||
relationSelectFieldItemsTmp.add(fieldItem);
|
||||
}
|
||||
}
|
||||
relationSelectFieldItems = relationSelectFieldItemsTmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
return relationSelectFieldItems;
|
||||
}
|
||||
}
|
||||
|
||||
public static class RelationFieldsPair {
|
||||
|
||||
private FieldItem leftFieldItem;
|
||||
private FieldItem rightFieldItem;
|
||||
|
||||
public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){
|
||||
this.leftFieldItem = leftFieldItem;
|
||||
this.rightFieldItem = rightFieldItem;
|
||||
}
|
||||
|
||||
public FieldItem getLeftFieldItem() {
|
||||
return leftFieldItem;
|
||||
}
|
||||
|
||||
public void setLeftFieldItem(FieldItem leftFieldItem) {
|
||||
this.leftFieldItem = leftFieldItem;
|
||||
}
|
||||
|
||||
public FieldItem getRightFieldItem() {
|
||||
return rightFieldItem;
|
||||
}
|
||||
|
||||
public void setRightFieldItem(FieldItem rightFieldItem) {
|
||||
this.rightFieldItem = rightFieldItem;
|
||||
}
|
||||
}
|
||||
|
||||
public static class FieldItem {
|
||||
|
||||
private String fieldName;
|
||||
private String expr;
|
||||
private List<ColumnItem> columnItems = new ArrayList<>();
|
||||
private List<String> owners = new ArrayList<>();
|
||||
|
||||
private boolean method;
|
||||
private boolean binaryOp;
|
||||
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
public void setFieldName(String fieldName) {
|
||||
this.fieldName = fieldName;
|
||||
}
|
||||
|
||||
public String getExpr() {
|
||||
return expr;
|
||||
}
|
||||
|
||||
public void setExpr(String expr) {
|
||||
this.expr = expr;
|
||||
}
|
||||
|
||||
public List<ColumnItem> getColumnItems() {
|
||||
return columnItems;
|
||||
}
|
||||
|
||||
public void setColumnItems(List<ColumnItem> columnItems) {
|
||||
this.columnItems = columnItems;
|
||||
}
|
||||
|
||||
public boolean isMethod() {
|
||||
return method;
|
||||
}
|
||||
|
||||
public void setMethod(boolean method) {
|
||||
this.method = method;
|
||||
}
|
||||
|
||||
public boolean isBinaryOp() {
|
||||
return binaryOp;
|
||||
}
|
||||
|
||||
public void setBinaryOp(boolean binaryOp) {
|
||||
this.binaryOp = binaryOp;
|
||||
}
|
||||
|
||||
public List<String> getOwners() {
|
||||
return owners;
|
||||
}
|
||||
|
||||
public void setOwners(List<String> owners) {
|
||||
this.owners = owners;
|
||||
}
|
||||
|
||||
public void addColumn(ColumnItem columnItem) {
|
||||
columnItems.add(columnItem);
|
||||
}
|
||||
|
||||
public ColumnItem getColumn() {
|
||||
if (!columnItems.isEmpty()) {
|
||||
return columnItems.get(0);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String getOwner() {
|
||||
if (!owners.isEmpty()) {
|
||||
return owners.get(0);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
FieldItem fieldItem = (FieldItem) o;
|
||||
|
||||
return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return fieldName != null ? fieldName.hashCode() : 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static class ColumnItem {
|
||||
|
||||
private String owner;
|
||||
private String columnName;
|
||||
|
||||
public String getOwner() {
|
||||
return owner;
|
||||
}
|
||||
|
||||
public void setOwner(String owner) {
|
||||
this.owner = owner;
|
||||
}
|
||||
|
||||
public String getColumnName() {
|
||||
return columnName;
|
||||
}
|
||||
|
||||
public void setColumnName(String columnName) {
|
||||
this.columnName = columnName;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.config;
|
||||
|
||||
import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
|
||||
import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.alibaba.fastsql.sql.SQLUtils;
|
||||
import com.alibaba.fastsql.sql.ast.SQLExpr;
|
||||
import com.alibaba.fastsql.sql.ast.expr.*;
|
||||
import com.alibaba.fastsql.sql.ast.statement.*;
|
||||
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
|
||||
import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
|
||||
import com.alibaba.fastsql.sql.parser.ParserException;
|
||||
import com.alibaba.fastsql.sql.parser.SQLStatementParser;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.RelationFieldsPair;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
|
||||
|
||||
/**
|
||||
* ES同步指定sql格式解析
|
||||
*
|
||||
* @author rewerma 2018-10-26 下午03:45:49
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class SqlParser {
|
||||
|
||||
/**
|
||||
* 解析sql
|
||||
*
|
||||
* @param sql sql
|
||||
* @return 视图对象
|
||||
*/
|
||||
public static SchemaItem parse(String sql) {
|
||||
try {
|
||||
SQLStatementParser parser = new MySqlStatementParser(sql);
|
||||
SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement();
|
||||
MySqlSelectQueryBlock sqlSelectQueryBlock = (MySqlSelectQueryBlock) statement.getSelect().getQuery();
|
||||
|
||||
SchemaItem schemaItem = new SchemaItem();
|
||||
schemaItem.setSql(SQLUtils.toMySqlString(sqlSelectQueryBlock));
|
||||
SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom();
|
||||
List<TableItem> tableItems = new ArrayList<>();
|
||||
SqlParser.visitSelectTable(schemaItem, sqlTableSource, tableItems, null);
|
||||
tableItems.forEach(tableItem -> schemaItem.getAliasTableItems().put(tableItem.getAlias(), tableItem));
|
||||
|
||||
List<FieldItem> fieldItems = collectSelectQueryFields(sqlSelectQueryBlock);
|
||||
fieldItems.forEach(fieldItem -> schemaItem.getSelectFields().put(fieldItem.getFieldName(), fieldItem));
|
||||
|
||||
schemaItem.init();
|
||||
|
||||
if (schemaItem.getAliasTableItems().isEmpty() || schemaItem.getSelectFields().isEmpty()) {
|
||||
throw new ParserException("Parse sql error");
|
||||
}
|
||||
return schemaItem;
|
||||
} catch (Exception e) {
|
||||
throw new ParserException();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 归集字段
|
||||
*
|
||||
* @param sqlSelectQueryBlock sqlSelectQueryBlock
|
||||
* @return 字段属性列表
|
||||
*/
|
||||
private static List<FieldItem> collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) {
|
||||
return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> {
|
||||
FieldItem fieldItem = new FieldItem();
|
||||
fieldItem.setFieldName(selectItem.getAlias());
|
||||
fieldItem.setExpr(selectItem.toString());
|
||||
visitColumn(selectItem.getExpr(), fieldItem);
|
||||
return fieldItem;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析字段
|
||||
*
|
||||
* @param expr sql expr
|
||||
* @param fieldItem 字段属性
|
||||
*/
|
||||
private static void visitColumn(SQLExpr expr, FieldItem fieldItem) {
|
||||
if (expr instanceof SQLIdentifierExpr) {
|
||||
// 无owner
|
||||
SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr;
|
||||
if (fieldItem.getFieldName() == null) {
|
||||
fieldItem.setFieldName(identifierExpr.getName());
|
||||
fieldItem.setExpr(identifierExpr.toString());
|
||||
}
|
||||
ColumnItem columnItem = new ColumnItem();
|
||||
columnItem.setColumnName(identifierExpr.getName());
|
||||
fieldItem.getOwners().add(null);
|
||||
fieldItem.addColumn(columnItem);
|
||||
} else if (expr instanceof SQLPropertyExpr) {
|
||||
// 有owner
|
||||
SQLPropertyExpr sqlPropertyExpr = (SQLPropertyExpr) expr;
|
||||
if (fieldItem.getFieldName() == null) {
|
||||
fieldItem.setFieldName(sqlPropertyExpr.getName());
|
||||
fieldItem.setExpr(sqlPropertyExpr.toString());
|
||||
}
|
||||
fieldItem.getOwners().add(sqlPropertyExpr.getOwnernName());
|
||||
ColumnItem columnItem = new ColumnItem();
|
||||
columnItem.setColumnName(sqlPropertyExpr.getName());
|
||||
columnItem.setOwner(sqlPropertyExpr.getOwnernName());
|
||||
fieldItem.addColumn(columnItem);
|
||||
} else if (expr instanceof SQLMethodInvokeExpr) {
|
||||
SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr) expr;
|
||||
fieldItem.setMethod(true);
|
||||
for (SQLExpr sqlExpr : methodInvokeExpr.getArguments()) {
|
||||
visitColumn(sqlExpr, fieldItem);
|
||||
}
|
||||
} else if (expr instanceof SQLBinaryOpExpr) {
|
||||
SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr;
|
||||
fieldItem.setBinaryOp(true);
|
||||
visitColumn(sqlBinaryOpExpr.getLeft(), fieldItem);
|
||||
visitColumn(sqlBinaryOpExpr.getRight(), fieldItem);
|
||||
} else if (expr instanceof SQLCaseExpr) {
|
||||
SQLCaseExpr sqlCaseExpr = (SQLCaseExpr) expr;
|
||||
fieldItem.setMethod(true);
|
||||
sqlCaseExpr.getItems().forEach(item -> visitColumn(item.getConditionExpr(), fieldItem));
|
||||
} else if (expr instanceof SQLCharExpr) {
|
||||
SQLCharExpr sqlCharExpr = (SQLCharExpr) expr;
|
||||
String owner = null;
|
||||
String columnName = null;
|
||||
if (sqlCharExpr.getParent() instanceof SQLCaseExpr.Item) {
|
||||
owner = ((SQLPropertyExpr) ((SQLCaseExpr.Item) sqlCharExpr.getParent()).getValueExpr()).getOwnernName();
|
||||
columnName = ((SQLPropertyExpr) ((SQLCaseExpr.Item) sqlCharExpr.getParent()).getValueExpr()).getName();
|
||||
}
|
||||
if (fieldItem.getFieldName() == null) {
|
||||
fieldItem.setFieldName(columnName);
|
||||
fieldItem.setExpr(sqlCharExpr.toString());
|
||||
}
|
||||
ColumnItem columnItem = new ColumnItem();
|
||||
columnItem.setColumnName(columnName);
|
||||
columnItem.setOwner(owner);
|
||||
fieldItem.getOwners().add(owner);
|
||||
fieldItem.addColumn(columnItem);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析表
|
||||
*
|
||||
* @param schemaItem 视图对象
|
||||
* @param sqlTableSource sqlTableSource
|
||||
* @param tableItems 表对象列表
|
||||
* @param tableItemTmp 表对象(临时)
|
||||
*/
|
||||
private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource,
|
||||
List<TableItem> tableItems, TableItem tableItemTmp) {
|
||||
if (sqlTableSource instanceof SQLExprTableSource) {
|
||||
SQLExprTableSource sqlExprTableSource = (SQLExprTableSource) sqlTableSource;
|
||||
TableItem tableItem;
|
||||
if (tableItemTmp != null) {
|
||||
tableItem = tableItemTmp;
|
||||
} else {
|
||||
tableItem = new TableItem(schemaItem);
|
||||
}
|
||||
tableItem.setSchema(sqlExprTableSource.getSchema());
|
||||
tableItem.setTableName(sqlExprTableSource.getTableName());
|
||||
if (tableItem.getAlias() == null) {
|
||||
tableItem.setAlias(sqlExprTableSource.getAlias());
|
||||
}
|
||||
if (tableItems.isEmpty()) {
|
||||
// 第一张表为主表
|
||||
tableItem.setMain(true);
|
||||
}
|
||||
tableItems.add(tableItem);
|
||||
} else if (sqlTableSource instanceof SQLJoinTableSource) {
|
||||
SQLJoinTableSource sqlJoinTableSource = (SQLJoinTableSource) sqlTableSource;
|
||||
SQLTableSource leftTableSource = sqlJoinTableSource.getLeft();
|
||||
visitSelectTable(schemaItem, leftTableSource, tableItems, null);
|
||||
SQLTableSource rightTableSource = sqlJoinTableSource.getRight();
|
||||
TableItem rightTableItem = new TableItem(schemaItem);
|
||||
// 解析on条件字段
|
||||
visitOnCondition(sqlJoinTableSource.getCondition(), rightTableItem);
|
||||
visitSelectTable(schemaItem, rightTableSource, tableItems, rightTableItem);
|
||||
|
||||
} else if (sqlTableSource instanceof SQLSubqueryTableSource) {
|
||||
SQLSubqueryTableSource subQueryTableSource = (SQLSubqueryTableSource) sqlTableSource;
|
||||
MySqlSelectQueryBlock sqlSelectQuery = (MySqlSelectQueryBlock) subQueryTableSource.getSelect().getQuery();
|
||||
TableItem tableItem;
|
||||
if (tableItemTmp != null) {
|
||||
tableItem = tableItemTmp;
|
||||
} else {
|
||||
tableItem = new TableItem(schemaItem);
|
||||
}
|
||||
tableItem.setAlias(subQueryTableSource.getAlias());
|
||||
tableItem.setSubQuerySql(SQLUtils.toMySqlString(sqlSelectQuery));
|
||||
tableItem.setSubQuery(true);
|
||||
tableItem.setSubQueryFields(collectSelectQueryFields(sqlSelectQuery));
|
||||
visitSelectTable(schemaItem, sqlSelectQuery.getFrom(), tableItems, tableItem);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析on条件
|
||||
*
|
||||
* @param expr sql expr
|
||||
* @param tableItem 表对象
|
||||
*/
|
||||
private static void visitOnCondition(SQLExpr expr, TableItem tableItem) {
|
||||
if (!(expr instanceof SQLBinaryOpExpr)) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr;
|
||||
if (sqlBinaryOpExpr.getOperator() == BooleanAnd) {
|
||||
visitOnCondition(sqlBinaryOpExpr.getLeft(), tableItem);
|
||||
visitOnCondition(sqlBinaryOpExpr.getRight(), tableItem);
|
||||
} else if (sqlBinaryOpExpr.getOperator() == Equality) {
|
||||
FieldItem leftFieldItem = new FieldItem();
|
||||
visitColumn(sqlBinaryOpExpr.getLeft(), leftFieldItem);
|
||||
if (leftFieldItem.getColumnItems().size() != 1 || leftFieldItem.isMethod() || leftFieldItem.isBinaryOp()) {
|
||||
throw new UnsupportedOperationException("Unsupported for complex of on-condition");
|
||||
}
|
||||
FieldItem rightFieldItem = new FieldItem();
|
||||
visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
|
||||
if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
|
||||
|| rightFieldItem.isBinaryOp()) {
|
||||
throw new UnsupportedOperationException("Unsupported for complex of on-condition");
|
||||
}
|
||||
tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unsupported for complex of on-condition");
|
||||
}
|
||||
}
|
||||
|
||||
public static MySqlSelectQueryBlock parseSQLSelectQueryBlock(String sql) {
|
||||
if (sql == null || "".equals(sql)) {
|
||||
return null;
|
||||
}
|
||||
SQLStatementParser parser = new MySqlStatementParser(sql);
|
||||
SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement();
|
||||
return (MySqlSelectQueryBlock) statement.getSelect().getQuery();
|
||||
}
|
||||
|
||||
public static String parse4SQLSelectItem(MySqlSelectQueryBlock sqlSelectQueryBlock) {
|
||||
List<SQLSelectItem> selectItems = sqlSelectQueryBlock.getSelectList();
|
||||
StringBuilder subSql = new StringBuilder();
|
||||
int i = 0;
|
||||
for (SQLSelectItem sqlSelectItem : selectItems) {
|
||||
if (i != 0) {
|
||||
subSql.append(",");
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
subSql.append(SQLUtils.toMySqlString(sqlSelectItem));
|
||||
}
|
||||
return subSql.toString();
|
||||
}
|
||||
|
||||
public static String parse4FromTableSource(MySqlSelectQueryBlock sqlSelectQueryBlock) {
|
||||
SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom();
|
||||
return SQLUtils.toMySqlString(sqlTableSource);
|
||||
}
|
||||
|
||||
public static String parse4WhereItem(MySqlSelectQueryBlock sqlSelectQueryBlock) {
|
||||
SQLExpr sqlExpr = sqlSelectQueryBlock.getWhere();
|
||||
if (sqlExpr != null) {
|
||||
return SQLUtils.toMySqlString(sqlExpr);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static String parse4GroupBy(MySqlSelectQueryBlock sqlSelectQueryBlock) {
|
||||
SQLSelectGroupByClause expr = sqlSelectQueryBlock.getGroupBy();
|
||||
if (expr != null) {
|
||||
return SQLUtils.toMySqlString(expr);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.monitor;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
|
||||
import org.apache.commons.io.monitor.FileAlterationMonitor;
|
||||
import org.apache.commons.io.monitor.FileAlterationObserver;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
|
||||
public class ESConfigMonitor {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ESConfigMonitor.class);
|
||||
|
||||
private static final String adapterName = "es";
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
private Properties envProperties;
|
||||
|
||||
private FileAlterationMonitor fileMonitor;
|
||||
|
||||
public void init(ESAdapter esAdapter, Properties envProperties) {
|
||||
this.esAdapter = esAdapter;
|
||||
this.envProperties = envProperties;
|
||||
File confDir = Util.getConfDirPath(adapterName);
|
||||
try {
|
||||
FileAlterationObserver observer = new FileAlterationObserver(confDir,
|
||||
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
|
||||
FileListener listener = new FileListener();
|
||||
observer.addListener(listener);
|
||||
fileMonitor = new FileAlterationMonitor(3000, observer);
|
||||
fileMonitor.start();
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public void destroy() {
|
||||
try {
|
||||
fileMonitor.stop();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private class FileListener extends FileAlterationListenerAdaptor {
|
||||
|
||||
@Override
|
||||
public void onFileCreate(File file) {
|
||||
super.onFileCreate(file);
|
||||
try {
|
||||
// 加载新增的配置文件
|
||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||
ESSyncConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
|
||||
if (config != null) {
|
||||
config.validate();
|
||||
addConfigToCache(file, config);
|
||||
logger.info("Add a new es mapping config: {} to canal adapter", file.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFileChange(File file) {
|
||||
super.onFileChange(file);
|
||||
|
||||
try {
|
||||
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
|
||||
// 加载配置文件
|
||||
String configContent = MappingConfigsLoader
|
||||
.loadConfig(adapterName + File.separator + file.getName());
|
||||
if (configContent == null) {
|
||||
onFileDelete(file);
|
||||
return;
|
||||
}
|
||||
ESSyncConfig config = YmlConfigBinder
|
||||
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
|
||||
if (config == null) {
|
||||
return;
|
||||
}
|
||||
config.validate();
|
||||
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
|
||||
deleteConfigFromCache(file);
|
||||
}
|
||||
addConfigToCache(file, config);
|
||||
|
||||
logger.info("Change a es mapping config: {} of canal adapter", file.getName());
|
||||
}else{
|
||||
onFileCreate(file);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFileDelete(File file) {
|
||||
super.onFileDelete(file);
|
||||
|
||||
try {
|
||||
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
|
||||
deleteConfigFromCache(file);
|
||||
|
||||
logger.info("Delete a es mapping config: {} of canal adapter", file.getName());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void addConfigToCache(File file, ESSyncConfig config) {
|
||||
esAdapter.getEsSyncConfig().put(file.getName(), config);
|
||||
esAdapter.addSyncConfigToCache(file.getName(),config);
|
||||
}
|
||||
|
||||
private void deleteConfigFromCache(File file) {
|
||||
esAdapter.getEsSyncConfig().remove(file.getName());
|
||||
for (Map<String, ESSyncConfig> configMap : esAdapter.getDbTableEsSyncConfig().values()) {
|
||||
if (configMap != null) {
|
||||
configMap.remove(file.getName());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.service;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.elasticsearch.action.bulk.BulkItemResponse;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.rest.RestStatus;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESBulkRequest;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESIndexRequest;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESSearchRequest;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESUpdateRequest;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
|
||||
import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
|
||||
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
|
||||
/**
|
||||
* ES ETL Service
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESEtlService extends AbstractEtlService {
|
||||
|
||||
private ESConnection esConnection;
|
||||
private ESTemplate esTemplate;
|
||||
private ESSyncConfig config;
|
||||
|
||||
public ESEtlService(ESConnection esConnection, ESSyncConfig config){
|
||||
super("ES", config);
|
||||
this.esConnection = esConnection;
|
||||
this.esTemplate = new ESTemplate(esConnection);
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public EtlResult importData(List<String> params) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
logger.info("start etl to import data to index: {}", mapping.get_index());
|
||||
String sql = mapping.getSql();
|
||||
return importData(sql, params);
|
||||
}
|
||||
|
||||
private void processFailBulkResponse(BulkResponse bulkResponse) {
|
||||
for (BulkItemResponse response : bulkResponse.getItems()) {
|
||||
if (!response.isFailed()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) {
|
||||
logger.warn(response.getFailureMessage());
|
||||
} else {
|
||||
logger.error("全量导入数据有误 {}", response.getFailureMessage());
|
||||
throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
|
||||
AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount,
|
||||
List<String> errMsg) {
|
||||
try {
|
||||
ESMapping mapping = (ESMapping) adapterMapping;
|
||||
Util.sqlRS(ds, sql, values, rs -> {
|
||||
int count = 0;
|
||||
try {
|
||||
ESBulkRequest esBulkRequest = this.esConnection.new ESBulkRequest();
|
||||
|
||||
long batchBegin = System.currentTimeMillis();
|
||||
while (rs.next()) {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
Object idVal = null;
|
||||
for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
|
||||
|
||||
String fieldName = fieldItem.getFieldName();
|
||||
if (mapping.getSkips().contains(fieldName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 如果是主键字段则不插入
|
||||
if (fieldItem.getFieldName().equals(mapping.get_id())) {
|
||||
idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
|
||||
} else {
|
||||
Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
|
||||
esFieldData.put(Util.cleanColumn(fieldName), val);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (!mapping.getRelations().isEmpty()) {
|
||||
mapping.getRelations().forEach((relationField, relationMapping) -> {
|
||||
Map<String, Object> relations = new HashMap<>();
|
||||
relations.put("name", relationMapping.getName());
|
||||
if (StringUtils.isNotEmpty(relationMapping.getParent())) {
|
||||
FieldItem parentFieldItem = mapping.getSchemaItem()
|
||||
.getSelectFields()
|
||||
.get(relationMapping.getParent());
|
||||
Object parentVal;
|
||||
try {
|
||||
parentVal = esTemplate.getValFromRS(mapping,
|
||||
rs,
|
||||
parentFieldItem.getFieldName(),
|
||||
parentFieldItem.getFieldName());
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
if (parentVal != null) {
|
||||
relations.put("parent", parentVal.toString());
|
||||
esFieldData.put("$parent_routing", parentVal.toString());
|
||||
|
||||
}
|
||||
}
|
||||
esFieldData.put(Util.cleanColumn(relationField), relations);
|
||||
});
|
||||
}
|
||||
|
||||
if (idVal != null) {
|
||||
String parentVal = (String) esFieldData.remove("$parent_routing");
|
||||
if (mapping.isUpsert()) {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(
|
||||
mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
|
||||
|
||||
if (StringUtils.isNotEmpty(parentVal)) {
|
||||
esUpdateRequest.setRouting(parentVal);
|
||||
}
|
||||
|
||||
esBulkRequest.add(esUpdateRequest);
|
||||
} else {
|
||||
ESIndexRequest esIndexRequest = this.esConnection.new ESIndexRequest(mapping
|
||||
.get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
|
||||
if (StringUtils.isNotEmpty(parentVal)) {
|
||||
esIndexRequest.setRouting(parentVal);
|
||||
}
|
||||
esBulkRequest.add(esIndexRequest);
|
||||
}
|
||||
} else {
|
||||
idVal = esFieldData.get(mapping.getPk());
|
||||
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
|
||||
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
|
||||
.size(10000);
|
||||
SearchResponse response = esSearchRequest.getResponse();
|
||||
for (SearchHit hit : response.getHits()) {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping
|
||||
.get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
|
||||
esBulkRequest.add(esUpdateRequest);
|
||||
}
|
||||
}
|
||||
|
||||
if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
|
||||
&& esBulkRequest.numberOfActions() > 0) {
|
||||
long esBatchBegin = System.currentTimeMillis();
|
||||
BulkResponse rp = esBulkRequest.bulk();
|
||||
if (rp.hasFailures()) {
|
||||
this.processFailBulkResponse(rp);
|
||||
}
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
|
||||
(System.currentTimeMillis() - batchBegin),
|
||||
(System.currentTimeMillis() - esBatchBegin),
|
||||
esBulkRequest.numberOfActions(),
|
||||
mapping.get_index());
|
||||
}
|
||||
batchBegin = System.currentTimeMillis();
|
||||
esBulkRequest.resetBulk();
|
||||
}
|
||||
count++;
|
||||
impCount.incrementAndGet();
|
||||
}
|
||||
|
||||
if (esBulkRequest.numberOfActions() > 0) {
|
||||
long esBatchBegin = System.currentTimeMillis();
|
||||
BulkResponse rp = esBulkRequest.bulk();
|
||||
if (rp.hasFailures()) {
|
||||
this.processFailBulkResponse(rp);
|
||||
}
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
|
||||
(System.currentTimeMillis() - batchBegin),
|
||||
(System.currentTimeMillis() - esBatchBegin),
|
||||
esBulkRequest.numberOfActions(),
|
||||
mapping.get_index());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return count;
|
||||
});
|
||||
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,877 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.service;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
|
||||
/**
|
||||
* ES 同步 Service
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESSyncService {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(ESSyncService.class);
|
||||
|
||||
private ESTemplate esTemplate;
|
||||
|
||||
public ESSyncService(ESTemplate esTemplate){
|
||||
this.esTemplate = esTemplate;
|
||||
}
|
||||
|
||||
public void sync(Collection<ESSyncConfig> esSyncConfigs, Dml dml) {
|
||||
long begin = System.currentTimeMillis();
|
||||
if (esSyncConfigs != null) {
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Destination: {}, database:{}, table:{}, type:{}, affected index count: {}",
|
||||
dml.getDestination(),
|
||||
dml.getDatabase(),
|
||||
dml.getTable(),
|
||||
dml.getType(),
|
||||
esSyncConfigs.size());
|
||||
}
|
||||
|
||||
for (ESSyncConfig config : esSyncConfigs) {
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Prepared to sync index: {}, destination: {}",
|
||||
config.getEsMapping().get_index(),
|
||||
dml.getDestination());
|
||||
}
|
||||
this.sync(config, dml);
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Sync completed: {}, destination: {}",
|
||||
config.getEsMapping().get_index(),
|
||||
dml.getDestination());
|
||||
}
|
||||
}
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Sync elapsed time: {} ms, affected indexes count：{}, destination: {}",
|
||||
(System.currentTimeMillis() - begin),
|
||||
esSyncConfigs.size(),
|
||||
dml.getDestination());
|
||||
}
|
||||
if (logger.isDebugEnabled()) {
|
||||
StringBuilder configIndexes = new StringBuilder();
|
||||
esSyncConfigs
|
||||
.forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().get_index()).append(" "));
|
||||
logger.debug("DML: {} \nAffected indexes: {}",
|
||||
JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue),
|
||||
configIndexes.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void sync(ESSyncConfig config, Dml dml) {
|
||||
try {
|
||||
// 如果是按时间戳定时更新则返回
|
||||
if (config.getEsMapping().isSyncByTimestamp()) {
|
||||
return;
|
||||
}
|
||||
|
||||
long begin = System.currentTimeMillis();
|
||||
|
||||
String type = dml.getType();
|
||||
if (type != null && type.equalsIgnoreCase("INSERT")) {
|
||||
insert(config, dml);
|
||||
} else if (type != null && type.equalsIgnoreCase("UPDATE")) {
|
||||
update(config, dml);
|
||||
} else if (type != null && type.equalsIgnoreCase("DELETE")) {
|
||||
delete(config, dml);
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}",
|
||||
(System.currentTimeMillis() - begin),
|
||||
dml.getDestination(),
|
||||
config.getEsMapping().get_index());
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 插入操作dml
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml数据
|
||||
*/
|
||||
private void insert(ESSyncConfig config, Dml dml) {
|
||||
List<Map<String, Object>> dataList = dml.getData();
|
||||
if (dataList == null || dataList.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
|
||||
for (Map<String, Object> data : dataList) {
|
||||
if (data == null || data.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
|
||||
// ------单表 & 所有字段都为简单字段------
|
||||
singleTableSimpleFiledInsert(config, dml, data);
|
||||
} else {
|
||||
// ------是主表 查询sql来插入------
|
||||
if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
|
||||
mainTableInsert(config, dml, data);
|
||||
}
|
||||
|
||||
// 从表的操作
|
||||
for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
|
||||
if (tableItem.isMain()) {
|
||||
continue;
|
||||
}
|
||||
if (!tableItem.getTableName().equals(dml.getTable())) {
|
||||
continue;
|
||||
}
|
||||
// 关联条件出现在主表查询条件是否为简单字段
|
||||
boolean allFieldsSimple = true;
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
|
||||
allFieldsSimple = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// 所有查询字段均为简单字段
|
||||
if (allFieldsSimple) {
|
||||
// 不是子查询
|
||||
if (!tableItem.isSubQuery()) {
|
||||
// ------关联表简单字段插入------
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
Object value = esTemplate.getValFromData(config.getEsMapping(),
|
||||
data,
|
||||
fieldItem.getFieldName(),
|
||||
fieldItem.getColumn().getColumnName());
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
|
||||
}
|
||||
|
||||
joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
|
||||
} else {
|
||||
// ------关联子表简单字段插入------
|
||||
subTableSimpleFieldOperation(config, dml, data, null, tableItem);
|
||||
}
|
||||
} else {
|
||||
// ------关联子表复杂字段插入 执行全sql更新es------
|
||||
wholeSqlOperation(config, dml, data, null, tableItem);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新操作dml
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml数据
|
||||
*/
|
||||
private void update(ESSyncConfig config, Dml dml) {
|
||||
List<Map<String, Object>> dataList = dml.getData();
|
||||
List<Map<String, Object>> oldList = dml.getOld();
|
||||
if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
|
||||
int i = 0;
|
||||
for (Map<String, Object> data : dataList) {
|
||||
Map<String, Object> old = oldList.get(i);
|
||||
if (data == null || data.isEmpty() || old == null || old.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
|
||||
// ------单表 & 所有字段都为简单字段------
|
||||
singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
|
||||
} else {
|
||||
// ------主表 查询sql来更新------
|
||||
if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||
FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName);
|
||||
|
||||
boolean idFieldSimple = true;
|
||||
if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) {
|
||||
idFieldSimple = false;
|
||||
}
|
||||
|
||||
boolean allUpdateFieldSimple = true;
|
||||
out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
for (ColumnItem columnItem : fieldItem.getColumnItems()) {
|
||||
if (schemaItem.getMainTable().getAlias().equals(columnItem.getOwner())) { // 如果是主表的字段
|
||||
if (old.containsKey(columnItem.getColumnName())) {
|
||||
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
|
||||
allUpdateFieldSimple = false;
|
||||
break out;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 不支持主键更新!!
|
||||
|
||||
// 判断是否有外键更新
|
||||
boolean fkChanged = false;
|
||||
for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
|
||||
if (tableItem.isMain()) {
|
||||
continue;
|
||||
}
|
||||
boolean changed = false;
|
||||
for (List<FieldItem> fieldItems : tableItem.getRelationTableFields().values()) {
|
||||
for (FieldItem fieldItem : fieldItems) {
|
||||
if (old.containsKey(fieldItem.getColumn().getColumnName())) {
|
||||
fkChanged = true;
|
||||
changed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// 如果外键有修改,则更新所对应该表的所有查询条件数据
|
||||
if (changed) {
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
fieldItem.getColumnItems()
|
||||
.forEach(columnItem -> old.put(columnItem.getColumnName(), null));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 判断主键和所更新的字段是否全为简单字段
|
||||
if (idFieldSimple && allUpdateFieldSimple && !fkChanged) {
|
||||
singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
|
||||
} else {
|
||||
mainTableUpdate(config, dml, data, old);
|
||||
}
|
||||
}
|
||||
|
||||
// 从表的操作
|
||||
for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
|
||||
if (tableItem.isMain()) {
|
||||
continue;
|
||||
}
|
||||
if (!tableItem.getTableName().equals(dml.getTable())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 关联条件出现在主表查询条件是否为简单字段
|
||||
boolean allFieldsSimple = true;
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
|
||||
allFieldsSimple = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 所有查询字段均为简单字段
|
||||
if (allFieldsSimple) {
|
||||
// 不是子查询
|
||||
if (!tableItem.isSubQuery()) {
|
||||
// ------关联表简单字段更新------
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
if (old.containsKey(fieldItem.getColumn().getColumnName())) {
|
||||
Object value = esTemplate.getValFromData(config.getEsMapping(),
|
||||
data,
|
||||
fieldItem.getFieldName(),
|
||||
fieldItem.getColumn().getColumnName());
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
|
||||
}
|
||||
}
|
||||
joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
|
||||
} else {
|
||||
// ------关联子表简单字段更新------
|
||||
subTableSimpleFieldOperation(config, dml, data, old, tableItem);
|
||||
}
|
||||
} else {
|
||||
// ------关联子表复杂字段更新 执行全sql更新es------
|
||||
wholeSqlOperation(config, dml, data, old, tableItem);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除操作dml
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml数据
|
||||
*/
|
||||
private void delete(ESSyncConfig config, Dml dml) {
|
||||
List<Map<String, Object>> dataList = dml.getData();
|
||||
if (dataList == null || dataList.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
|
||||
|
||||
for (Map<String, Object> data : dataList) {
|
||||
if (data == null || data.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
|
||||
// ------是主表------
|
||||
if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
|
||||
if (mapping.get_id() != null) {
|
||||
FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
|
||||
// 主键为简单字段
|
||||
if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
|
||||
Object idVal = esTemplate.getValFromData(mapping,
|
||||
data,
|
||||
idFieldItem.getFieldName(),
|
||||
idFieldItem.getColumn().getColumnName());
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
idVal);
|
||||
}
|
||||
esTemplate.delete(mapping, idVal, null);
|
||||
} else {
|
||||
// ------主键带函数, 查询sql获取主键删除------
|
||||
// FIXME 删除时反查sql为空记录, 无法获获取 id field 值
|
||||
mainTableDelete(config, dml, data);
|
||||
}
|
||||
} else {
|
||||
FieldItem pkFieldItem = schemaItem.getIdFieldItem(mapping);
|
||||
if (!pkFieldItem.isMethod() && !pkFieldItem.isBinaryOp()) {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
Object pkVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
pkVal);
|
||||
}
|
||||
esFieldData.remove(pkFieldItem.getFieldName());
|
||||
esFieldData.keySet().forEach(key -> esFieldData.put(key, null));
|
||||
esTemplate.delete(mapping, pkVal, esFieldData);
|
||||
} else {
|
||||
// ------主键带函数, 查询sql获取主键删除------
|
||||
mainTableDelete(config, dml, data);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 从表的操作
|
||||
for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
|
||||
if (tableItem.isMain()) {
|
||||
continue;
|
||||
}
|
||||
if (!tableItem.getTableName().equals(dml.getTable())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 关联条件出现在主表查询条件是否为简单字段
|
||||
boolean allFieldsSimple = true;
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
|
||||
allFieldsSimple = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 所有查询字段均为简单字段
|
||||
if (allFieldsSimple) {
|
||||
// 不是子查询
|
||||
if (!tableItem.isSubQuery()) {
|
||||
// ------关联表简单字段更新为null------
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), null);
|
||||
}
|
||||
joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
|
||||
} else {
|
||||
// ------关联子表简单字段更新------
|
||||
subTableSimpleFieldOperation(config, dml, data, null, tableItem);
|
||||
}
|
||||
} else {
|
||||
// ------关联子表复杂字段更新 执行全sql更新es------
|
||||
wholeSqlOperation(config, dml, data, null, tableItem);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 单表简单字段insert
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml信息
|
||||
* @param data 单行dml数据
|
||||
*/
|
||||
private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Single table insert to es index, destination:{}, table: {}, index: {}, id: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
idVal);
|
||||
}
|
||||
esTemplate.insert(mapping, idVal, esFieldData);
|
||||
}
|
||||
|
||||
/**
|
||||
* 主表(单表)复杂字段insert
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml信息
|
||||
* @param data 单行dml数据
|
||||
*/
|
||||
private void mainTableInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
String sql = mapping.getSql();
|
||||
String condition = ESSyncUtil.pkConditionSql(mapping, data);
|
||||
sql = ESSyncUtil.appendCondition(sql, condition);
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Main table insert to es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
sql.replace("\n", " "));
|
||||
}
|
||||
Util.sqlRS(ds, sql, rs -> {
|
||||
try {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace(
|
||||
"Main table insert to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
idVal);
|
||||
}
|
||||
esTemplate.insert(mapping, idVal, esFieldData);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void mainTableDelete(ESSyncConfig config, Dml dml, Map<String, Object> data) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
String sql = mapping.getSql();
|
||||
String condition = ESSyncUtil.pkConditionSql(mapping, data);
|
||||
sql = ESSyncUtil.appendCondition(sql, condition);
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
sql.replace("\n", " "));
|
||||
}
|
||||
Util.sqlRS(ds, sql, rs -> {
|
||||
try {
|
||||
Map<String, Object> esFieldData = null;
|
||||
if (mapping.getPk() != null) {
|
||||
esFieldData = new LinkedHashMap<>();
|
||||
esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
|
||||
esFieldData.remove(mapping.getPk());
|
||||
for (String key : esFieldData.keySet()) {
|
||||
esFieldData.put(Util.cleanColumn(key), null);
|
||||
}
|
||||
}
|
||||
while (rs.next()) {
|
||||
Object idVal = esTemplate.getIdValFromRS(mapping, rs);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace(
|
||||
"Main table delete to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
idVal);
|
||||
}
|
||||
esTemplate.delete(mapping, idVal, esFieldData);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 关联表主表简单字段operation
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml信息
|
||||
* @param data 单行dml数据
|
||||
* @param tableItem 当前表配置
|
||||
*/
|
||||
private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data,
|
||||
TableItem tableItem, Map<String, Object> esFieldData) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
|
||||
Map<String, Object> paramsTmp = new LinkedHashMap<>();
|
||||
for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
|
||||
for (FieldItem fieldItem : entry.getValue()) {
|
||||
if (fieldItem.getColumnItems().size() == 1) {
|
||||
Object value = esTemplate.getValFromData(mapping,
|
||||
data,
|
||||
fieldItem.getFieldName(),
|
||||
entry.getKey().getColumn().getColumnName());
|
||||
|
||||
String fieldName = fieldItem.getFieldName();
|
||||
// 判断是否是主键
|
||||
if (fieldName.equals(mapping.get_id())) {
|
||||
fieldName = "_id";
|
||||
}
|
||||
paramsTmp.put(fieldName, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index());
|
||||
}
|
||||
esTemplate.updateByQuery(config, paramsTmp, esFieldData);
|
||||
}
|
||||
|
||||
/**
|
||||
* 关联子查询, 主表简单字段operation
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml信息
|
||||
* @param data 单行dml数据
|
||||
* @param old 单行old数据
|
||||
* @param tableItem 当前表配置
|
||||
*/
|
||||
private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data,
|
||||
Map<String, Object> old, TableItem tableItem) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
|
||||
MySqlSelectQueryBlock queryBlock = SqlParser.parseSQLSelectQueryBlock(tableItem.getSubQuerySql());
|
||||
StringBuilder sql = new StringBuilder();
|
||||
sql.append("SELECT ")
|
||||
.append(SqlParser.parse4SQLSelectItem(queryBlock))
|
||||
.append(" FROM ")
|
||||
.append(SqlParser.parse4FromTableSource(queryBlock));
|
||||
|
||||
String whereSql = SqlParser.parse4WhereItem(queryBlock);
|
||||
if (whereSql != null) {
|
||||
sql.append(" WHERE ").append(whereSql);
|
||||
} else {
|
||||
sql.append(" WHERE 1=1 ");
|
||||
}
|
||||
|
||||
List<Object> values = new ArrayList<>();
|
||||
for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
|
||||
String columnName = fkFieldItem.getColumn().getColumnName();
|
||||
Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
|
||||
sql.append(" AND ").append(columnName).append("=? ");
|
||||
values.add(value);
|
||||
}
|
||||
|
||||
String groupSql = SqlParser.parse4GroupBy(queryBlock);
|
||||
if (groupSql != null) {
|
||||
sql.append(groupSql);
|
||||
}
|
||||
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
sql.toString().replace("\n", " "));
|
||||
}
|
||||
Util.sqlRS(ds, sql.toString(), values, rs -> {
|
||||
try {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
if (old != null) {
|
||||
out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
|
||||
for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
|
||||
if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
|
||||
for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
|
||||
if (old.containsKey(columnItem.getColumnName())) {
|
||||
Object val = esTemplate.getValFromRS(mapping,
|
||||
rs,
|
||||
fieldItem.getFieldName(),
|
||||
fieldItem.getColumn().getColumnName());
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
|
||||
break out;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Object val = esTemplate.getValFromRS(mapping,
|
||||
rs,
|
||||
fieldItem.getFieldName(),
|
||||
fieldItem.getColumn().getColumnName());
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> paramsTmp = new LinkedHashMap<>();
|
||||
for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
|
||||
for (FieldItem fieldItem : entry.getValue()) {
|
||||
if (fieldItem.getColumnItems().size() == 1) {
|
||||
Object value = esTemplate.getValFromRS(mapping,
|
||||
rs,
|
||||
fieldItem.getFieldName(),
|
||||
entry.getKey().getColumn().getColumnName());
|
||||
String fieldName = fieldItem.getFieldName();
|
||||
// 判断是否是主键
|
||||
if (fieldName.equals(mapping.get_id())) {
|
||||
fieldName = "_id";
|
||||
}
|
||||
paramsTmp.put(fieldName, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index());
|
||||
}
|
||||
esTemplate.updateByQuery(config, paramsTmp, esFieldData);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 关联(子查询), 主表复杂字段operation, 全sql执行
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml信息
|
||||
* @param data 单行dml数据
|
||||
* @param tableItem 当前表配置
|
||||
*/
|
||||
private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old,
|
||||
TableItem tableItem) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
// 防止最后出现groupby 导致sql解析异常
|
||||
String[] sqlSplit = mapping.getSql().split("GROUP\\ BY(?!(.*)ON)");
|
||||
String sqlNoWhere = sqlSplit[0];
|
||||
|
||||
String sqlGroupBy = "";
|
||||
|
||||
if (sqlSplit.length > 1) {
|
||||
sqlGroupBy = "GROUP BY " + sqlSplit[1];
|
||||
}
|
||||
|
||||
StringBuilder sql = new StringBuilder(sqlNoWhere + " WHERE ");
|
||||
|
||||
for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
|
||||
String columnName = fkFieldItem.getColumn().getColumnName();
|
||||
Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
|
||||
ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName);
|
||||
}
|
||||
int len = sql.length();
|
||||
sql.delete(len - 5, len);
|
||||
sql.append(sqlGroupBy);
|
||||
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
sql.toString().replace("\n", " "));
|
||||
}
|
||||
Util.sqlRS(ds, sql.toString(), rs -> {
|
||||
try {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
|
||||
if (old != null) {
|
||||
// 从表子查询
|
||||
out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
|
||||
for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
|
||||
if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
|
||||
for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
|
||||
if (old.containsKey(columnItem.getColumnName())) {
|
||||
Object val = esTemplate.getValFromRS(mapping,
|
||||
rs,
|
||||
fieldItem.getFieldName(),
|
||||
fieldItem.getFieldName());
|
||||
esFieldData.put(fieldItem.getFieldName(), val);
|
||||
break out;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 从表非子查询
|
||||
for (FieldItem fieldItem1 : tableItem.getRelationSelectFieldItems()) {
|
||||
if (fieldItem1.equals(fieldItem)) {
|
||||
for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
|
||||
if (old.containsKey(columnItem.getColumnName())) {
|
||||
Object val = esTemplate.getValFromRS(mapping,
|
||||
rs,
|
||||
fieldItem.getFieldName(),
|
||||
fieldItem.getFieldName());
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Object val = esTemplate
|
||||
.getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> paramsTmp = new LinkedHashMap<>();
|
||||
for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
|
||||
for (FieldItem fieldItem : entry.getValue()) {
|
||||
Object value = esTemplate
|
||||
.getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
|
||||
String fieldName = fieldItem.getFieldName();
|
||||
// 判断是否是主键
|
||||
if (fieldName.equals(mapping.get_id())) {
|
||||
fieldName = "_id";
|
||||
}
|
||||
paramsTmp.put(fieldName, value);
|
||||
}
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.trace(
|
||||
"Join table update es index by query whole sql, destination:{}, table: {}, index: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index());
|
||||
}
|
||||
esTemplate.updateByQuery(config, paramsTmp, esFieldData);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 单表简单字段update
|
||||
*
|
||||
* @param config es配置
|
||||
* @param owner 所属表
|
||||
* @param dml dml信息
|
||||
* @param data 单行data数据
|
||||
* @param old 单行old数据
|
||||
*/
|
||||
private void singleTableSimpleFiledUpdate(ESSyncConfig config, String owner, Dml dml, Map<String, Object> data,
|
||||
Map<String, Object> old) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
|
||||
Object idVal = esTemplate.getESDataFromDmlData(mapping, owner, data, old, esFieldData);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
idVal);
|
||||
}
|
||||
esTemplate.update(mapping, idVal, esFieldData);
|
||||
}
|
||||
|
||||
/**
|
||||
* 主表(单表)复杂字段update
|
||||
*
|
||||
* @param config es配置
|
||||
* @param dml dml信息
|
||||
* @param data 单行dml数据
|
||||
*/
|
||||
private void mainTableUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old) {
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
String sql = mapping.getSql();
|
||||
String condition = ESSyncUtil.pkConditionSql(mapping, data);
|
||||
sql = ESSyncUtil.appendCondition(sql, condition);
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Main table update to es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
sql.replace("\n", " "));
|
||||
}
|
||||
Util.sqlRS(ds, sql, rs -> {
|
||||
try {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData);
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace(
|
||||
"Main table update to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
|
||||
config.getDestination(),
|
||||
dml.getTable(),
|
||||
mapping.get_index(),
|
||||
idVal);
|
||||
}
|
||||
esTemplate.update(mapping, idVal, esFieldData);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交批次
|
||||
*/
|
||||
public void commit() {
|
||||
esTemplate.commit();
|
||||
}
|
||||
}
|
||||
@@ -1,469 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.support;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.bulk.BulkRequest;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.delete.DeleteRequestBuilder;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.index.IndexRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequestBuilder;
|
||||
import org.elasticsearch.client.*;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.TransportAddress;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.elasticsearch.transport.client.PreBuiltTransportClient;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* ES 连接器, Transport Rest 两种方式
|
||||
*
|
||||
* @author rewerma 2019-08-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESConnection {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
|
||||
|
||||
public enum ESClientMode {
|
||||
TRANSPORT, REST
|
||||
}
|
||||
|
||||
private ESClientMode mode;
|
||||
|
||||
private TransportClient transportClient;
|
||||
|
||||
private RestHighLevelClient restHighLevelClient;
|
||||
|
||||
public ESConnection(String[] hosts, Map<String, String> properties, ESClientMode mode) throws UnknownHostException{
|
||||
this.mode = mode;
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
Settings.Builder settingBuilder = Settings.builder();
|
||||
// properties.forEach(settingBuilder::put);
|
||||
settingBuilder.put("cluster.name", properties.get("properties"));
|
||||
Settings settings = settingBuilder.build();
|
||||
transportClient = new PreBuiltTransportClient(settings);
|
||||
for (String host : hosts) {
|
||||
int i = host.indexOf(":");
|
||||
transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
|
||||
Integer.parseInt(host.substring(i + 1))));
|
||||
}
|
||||
} else {
|
||||
HttpHost[] httpHosts = new HttpHost[hosts.length];
|
||||
for (int i = 0; i < hosts.length; i++) {
|
||||
String host = hosts[i];
|
||||
int j = host.indexOf(":");
|
||||
HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
|
||||
Integer.parseInt(host.substring(j + 1)));
|
||||
httpHosts[i] = httpHost;
|
||||
}
|
||||
RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
|
||||
String nameAndPwd = properties.get("security.auth");
|
||||
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
|
||||
String[] nameAndPwdArr = nameAndPwd.split(":");
|
||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY,
|
||||
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
|
||||
restClientBuilder.setHttpClientConfigCallback(
|
||||
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
|
||||
}
|
||||
restHighLevelClient = new RestHighLevelClient(restClientBuilder);
|
||||
}
|
||||
}
|
||||
|
||||
public void close() {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
transportClient.close();
|
||||
} else {
|
||||
try {
|
||||
restHighLevelClient.close();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public MappingMetaData getMapping(String index, String type) {
|
||||
MappingMetaData mappingMetaData = null;
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
ImmutableOpenMap<String, MappingMetaData> mappings;
|
||||
try {
|
||||
mappings = transportClient.admin()
|
||||
.cluster()
|
||||
.prepareState()
|
||||
.execute()
|
||||
.actionGet()
|
||||
.getState()
|
||||
.getMetaData()
|
||||
.getIndices()
|
||||
.get(index)
|
||||
.getMappings();
|
||||
} catch (NullPointerException e) {
|
||||
throw new IllegalArgumentException("Not found the mapping info of index: " + index);
|
||||
}
|
||||
mappingMetaData = mappings.get(type);
|
||||
|
||||
} else {
|
||||
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings;
|
||||
try {
|
||||
GetMappingsRequest request = new GetMappingsRequest();
|
||||
request.indices(index);
|
||||
GetMappingsResponse response;
|
||||
// try {
|
||||
// response = restHighLevelClient
|
||||
// .indices()
|
||||
// .getMapping(request, RequestOptions.DEFAULT);
|
||||
// // 6.4以下版本直接使用该接口会报错
|
||||
// } catch (Exception e) {
|
||||
// logger.warn("Low ElasticSearch version for getMapping");
|
||||
response = RestHighLevelClientExt.getMapping(restHighLevelClient, request, RequestOptions.DEFAULT);
|
||||
// }
|
||||
|
||||
mappings = response.mappings();
|
||||
} catch (NullPointerException e) {
|
||||
throw new IllegalArgumentException("Not found the mapping info of index: " + index);
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
mappingMetaData = mappings.get(index).get(type);
|
||||
}
|
||||
return mappingMetaData;
|
||||
}
|
||||
|
||||
public class ESIndexRequest {
|
||||
|
||||
private IndexRequestBuilder indexRequestBuilder;
|
||||
|
||||
private IndexRequest indexRequest;
|
||||
|
||||
public ESIndexRequest(String index, String type, String id){
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
indexRequestBuilder = transportClient.prepareIndex(index, type, id);
|
||||
} else {
|
||||
indexRequest = new IndexRequest(index, type, id);
|
||||
}
|
||||
}
|
||||
|
||||
public ESIndexRequest setSource(Map<String, ?> source) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
indexRequestBuilder.setSource(source);
|
||||
} else {
|
||||
indexRequest.source(source);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public ESIndexRequest setRouting(String routing) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
indexRequestBuilder.setRouting(routing);
|
||||
} else {
|
||||
indexRequest.routing(routing);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public IndexRequestBuilder getIndexRequestBuilder() {
|
||||
return indexRequestBuilder;
|
||||
}
|
||||
|
||||
public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) {
|
||||
this.indexRequestBuilder = indexRequestBuilder;
|
||||
}
|
||||
|
||||
public IndexRequest getIndexRequest() {
|
||||
return indexRequest;
|
||||
}
|
||||
|
||||
public void setIndexRequest(IndexRequest indexRequest) {
|
||||
this.indexRequest = indexRequest;
|
||||
}
|
||||
}
|
||||
|
||||
public class ESUpdateRequest {
|
||||
|
||||
private UpdateRequestBuilder updateRequestBuilder;
|
||||
|
||||
private UpdateRequest updateRequest;
|
||||
|
||||
public ESUpdateRequest(String index, String type, String id){
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
updateRequestBuilder = transportClient.prepareUpdate(index, type, id);
|
||||
} else {
|
||||
updateRequest = new UpdateRequest(index, type, id);
|
||||
}
|
||||
}
|
||||
|
||||
public ESUpdateRequest setDoc(Map source) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
updateRequestBuilder.setDoc(source);
|
||||
} else {
|
||||
updateRequest.doc(source);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public ESUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
updateRequestBuilder.setDocAsUpsert(shouldUpsertDoc);
|
||||
} else {
|
||||
updateRequest.docAsUpsert(shouldUpsertDoc);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public ESUpdateRequest setRouting(String routing) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
updateRequestBuilder.setRouting(routing);
|
||||
} else {
|
||||
updateRequest.routing(routing);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public UpdateRequestBuilder getUpdateRequestBuilder() {
|
||||
return updateRequestBuilder;
|
||||
}
|
||||
|
||||
public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) {
|
||||
this.updateRequestBuilder = updateRequestBuilder;
|
||||
}
|
||||
|
||||
public UpdateRequest getUpdateRequest() {
|
||||
return updateRequest;
|
||||
}
|
||||
|
||||
public void setUpdateRequest(UpdateRequest updateRequest) {
|
||||
this.updateRequest = updateRequest;
|
||||
}
|
||||
}
|
||||
|
||||
public class ESDeleteRequest {
|
||||
|
||||
private DeleteRequestBuilder deleteRequestBuilder;
|
||||
|
||||
private DeleteRequest deleteRequest;
|
||||
|
||||
public ESDeleteRequest(String index, String type, String id){
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
deleteRequestBuilder = transportClient.prepareDelete(index, type, id);
|
||||
} else {
|
||||
deleteRequest = new DeleteRequest(index, type, id);
|
||||
}
|
||||
}
|
||||
|
||||
public DeleteRequestBuilder getDeleteRequestBuilder() {
|
||||
return deleteRequestBuilder;
|
||||
}
|
||||
|
||||
public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) {
|
||||
this.deleteRequestBuilder = deleteRequestBuilder;
|
||||
}
|
||||
|
||||
public DeleteRequest getDeleteRequest() {
|
||||
return deleteRequest;
|
||||
}
|
||||
|
||||
public void setDeleteRequest(DeleteRequest deleteRequest) {
|
||||
this.deleteRequest = deleteRequest;
|
||||
}
|
||||
}
|
||||
|
||||
public class ESSearchRequest {
|
||||
|
||||
private SearchRequestBuilder searchRequestBuilder;
|
||||
|
||||
private SearchRequest searchRequest;
|
||||
|
||||
private SearchSourceBuilder sourceBuilder;
|
||||
|
||||
public ESSearchRequest(String index, String... types){
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
searchRequestBuilder = transportClient.prepareSearch(index).setTypes(types);
|
||||
} else {
|
||||
searchRequest = new SearchRequest(index).types(types);
|
||||
sourceBuilder = new SearchSourceBuilder();
|
||||
}
|
||||
}
|
||||
|
||||
public ESSearchRequest setQuery(QueryBuilder queryBuilder) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
searchRequestBuilder.setQuery(queryBuilder);
|
||||
} else {
|
||||
sourceBuilder.query(queryBuilder);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public ESSearchRequest size(int size) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
searchRequestBuilder.setSize(size);
|
||||
} else {
|
||||
sourceBuilder.size(size);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public SearchResponse getResponse() {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
return searchRequestBuilder.get();
|
||||
} else {
|
||||
searchRequest.source(sourceBuilder);
|
||||
try {
|
||||
return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public SearchRequestBuilder getSearchRequestBuilder() {
|
||||
return searchRequestBuilder;
|
||||
}
|
||||
|
||||
public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) {
|
||||
this.searchRequestBuilder = searchRequestBuilder;
|
||||
}
|
||||
|
||||
public SearchRequest getSearchRequest() {
|
||||
return searchRequest;
|
||||
}
|
||||
|
||||
public void setSearchRequest(SearchRequest searchRequest) {
|
||||
this.searchRequest = searchRequest;
|
||||
}
|
||||
}
|
||||
|
||||
public class ESBulkRequest {
|
||||
|
||||
private BulkRequestBuilder bulkRequestBuilder;
|
||||
|
||||
private BulkRequest bulkRequest;
|
||||
|
||||
public ESBulkRequest(){
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
bulkRequestBuilder = transportClient.prepareBulk();
|
||||
} else {
|
||||
bulkRequest = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
public void resetBulk() {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
bulkRequestBuilder = transportClient.prepareBulk();
|
||||
} else {
|
||||
bulkRequest = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
public ESBulkRequest add(ESIndexRequest esIndexRequest) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
bulkRequestBuilder.add(esIndexRequest.indexRequestBuilder);
|
||||
} else {
|
||||
bulkRequest.add(esIndexRequest.indexRequest);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public ESBulkRequest add(ESUpdateRequest esUpdateRequest) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
bulkRequestBuilder.add(esUpdateRequest.updateRequestBuilder);
|
||||
} else {
|
||||
bulkRequest.add(esUpdateRequest.updateRequest);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public ESBulkRequest add(ESDeleteRequest esDeleteRequest) {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
bulkRequestBuilder.add(esDeleteRequest.deleteRequestBuilder);
|
||||
} else {
|
||||
bulkRequest.add(esDeleteRequest.deleteRequest);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public int numberOfActions() {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
return bulkRequestBuilder.numberOfActions();
|
||||
} else {
|
||||
return bulkRequest.numberOfActions();
|
||||
}
|
||||
}
|
||||
|
||||
public BulkResponse bulk() {
|
||||
if (mode == ESClientMode.TRANSPORT) {
|
||||
return bulkRequestBuilder.execute().actionGet();
|
||||
} else {
|
||||
try {
|
||||
return restHighLevelClient.bulk(bulkRequest);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public BulkRequestBuilder getBulkRequestBuilder() {
|
||||
return bulkRequestBuilder;
|
||||
}
|
||||
|
||||
public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) {
|
||||
this.bulkRequestBuilder = bulkRequestBuilder;
|
||||
}
|
||||
|
||||
public BulkRequest getBulkRequest() {
|
||||
return bulkRequest;
|
||||
}
|
||||
|
||||
public void setBulkRequest(BulkRequest bulkRequest) {
|
||||
this.bulkRequest = bulkRequest;
|
||||
}
|
||||
}
|
||||
|
||||
// ------ get/set ------
|
||||
public ESClientMode getMode() {
|
||||
return mode;
|
||||
}
|
||||
|
||||
public void setMode(ESClientMode mode) {
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
public TransportClient getTransportClient() {
|
||||
return transportClient;
|
||||
}
|
||||
|
||||
public void setTransportClient(TransportClient transportClient) {
|
||||
this.transportClient = transportClient;
|
||||
}
|
||||
|
||||
public RestHighLevelClient getRestHighLevelClient() {
|
||||
return restHighLevelClient;
|
||||
}
|
||||
|
||||
public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
|
||||
this.restHighLevelClient = restHighLevelClient;
|
||||
}
|
||||
}
|
||||
@@ -1,309 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.support;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Blob;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.joda.time.DateTime;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
|
||||
/**
|
||||
* ES 同步工具同类
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESSyncUtil {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class);
|
||||
|
||||
public static Object convertToEsObj(Object val, String fieldInfo) {
|
||||
if (val == null) {
|
||||
return null;
|
||||
}
|
||||
if (fieldInfo.startsWith("array:")) {
|
||||
String separator = fieldInfo.substring("array:".length()).trim();
|
||||
String[] values = val.toString().split(separator);
|
||||
return Arrays.asList(values);
|
||||
} else if (fieldInfo.startsWith("object")) {
|
||||
if (val instanceof String){
|
||||
return JSON.parse(val.toString());
|
||||
}
|
||||
return JSON.parse(new String((byte[])val));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 类型转换为Mapping中对应的类型
|
||||
*/
|
||||
public static Object typeConvert(Object val, String esType) {
|
||||
if (val == null) {
|
||||
return null;
|
||||
}
|
||||
if (esType == null) {
|
||||
return val;
|
||||
}
|
||||
Object res = null;
|
||||
if ("integer".equals(esType)) {
|
||||
if (val instanceof Number) {
|
||||
res = ((Number) val).intValue();
|
||||
} else {
|
||||
res = Integer.parseInt(val.toString());
|
||||
}
|
||||
} else if ("long".equals(esType)) {
|
||||
if (val instanceof Number) {
|
||||
res = ((Number) val).longValue();
|
||||
} else {
|
||||
res = Long.parseLong(val.toString());
|
||||
}
|
||||
} else if ("short".equals(esType)) {
|
||||
if (val instanceof Number) {
|
||||
res = ((Number) val).shortValue();
|
||||
} else {
|
||||
res = Short.parseShort(val.toString());
|
||||
}
|
||||
} else if ("byte".equals(esType)) {
|
||||
if (val instanceof Number) {
|
||||
res = ((Number) val).byteValue();
|
||||
} else {
|
||||
res = Byte.parseByte(val.toString());
|
||||
}
|
||||
} else if ("double".equals(esType)) {
|
||||
if (val instanceof Number) {
|
||||
res = ((Number) val).doubleValue();
|
||||
} else {
|
||||
res = Double.parseDouble(val.toString());
|
||||
}
|
||||
} else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) {
|
||||
if (val instanceof Number) {
|
||||
res = ((Number) val).floatValue();
|
||||
} else {
|
||||
res = Float.parseFloat(val.toString());
|
||||
}
|
||||
} else if ("boolean".equals(esType)) {
|
||||
if (val instanceof Boolean) {
|
||||
res = val;
|
||||
} else if (val instanceof Number) {
|
||||
int v = ((Number) val).intValue();
|
||||
res = v != 0;
|
||||
} else {
|
||||
res = Boolean.parseBoolean(val.toString());
|
||||
}
|
||||
} else if ("date".equals(esType)) {
|
||||
if (val instanceof java.sql.Time) {
|
||||
DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
|
||||
if (dateTime.getMillisOfSecond() != 0) {
|
||||
res = dateTime.toString("HH:mm:ss.SSS");
|
||||
} else {
|
||||
res = dateTime.toString("HH:mm:ss");
|
||||
}
|
||||
} else if (val instanceof java.sql.Timestamp) {
|
||||
DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
|
||||
if (dateTime.getMillisOfSecond() != 0) {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
|
||||
} else {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
|
||||
}
|
||||
} else if (val instanceof java.sql.Date || val instanceof Date) {
|
||||
DateTime dateTime;
|
||||
if (val instanceof java.sql.Date) {
|
||||
dateTime = new DateTime(((java.sql.Date) val).getTime());
|
||||
} else {
|
||||
dateTime = new DateTime(((Date) val).getTime());
|
||||
}
|
||||
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
|
||||
&& dateTime.getMillisOfSecond() == 0) {
|
||||
res = dateTime.toString("yyyy-MM-dd");
|
||||
} else {
|
||||
if (dateTime.getMillisOfSecond() != 0) {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
|
||||
} else {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
|
||||
}
|
||||
}
|
||||
} else if (val instanceof Long) {
|
||||
DateTime dateTime = new DateTime(((Long) val).longValue());
|
||||
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
|
||||
&& dateTime.getMillisOfSecond() == 0) {
|
||||
res = dateTime.toString("yyyy-MM-dd");
|
||||
} else if (dateTime.getMillisOfSecond() != 0) {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
|
||||
} else {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
|
||||
}
|
||||
} else if (val instanceof String) {
|
||||
String v = ((String) val).trim();
|
||||
if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
|
||||
&& v.charAt(13) == ':' && v.charAt(16) == ':') {
|
||||
String dt = v.substring(0, 10) + "T" + v.substring(11);
|
||||
Date date = Util.parseDate(dt);
|
||||
if (date != null) {
|
||||
DateTime dateTime = new DateTime(date);
|
||||
if (dateTime.getMillisOfSecond() != 0) {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
|
||||
} else {
|
||||
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
|
||||
}
|
||||
}
|
||||
} else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
|
||||
Date date = Util.parseDate(v);
|
||||
if (date != null) {
|
||||
DateTime dateTime = new DateTime(date);
|
||||
res = dateTime.toString("yyyy-MM-dd");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if ("binary".equals(esType)) {
|
||||
if (val instanceof byte[]) {
|
||||
Base64 base64 = new Base64();
|
||||
res = base64.encodeAsString((byte[]) val);
|
||||
} else if (val instanceof Blob) {
|
||||
byte[] b = blobToBytes((Blob) val);
|
||||
Base64 base64 = new Base64();
|
||||
res = base64.encodeAsString(b);
|
||||
} else if (val instanceof String) {
|
||||
// 对应canal中的单字节编码
|
||||
byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
|
||||
Base64 base64 = new Base64();
|
||||
res = base64.encodeAsString(b);
|
||||
}
|
||||
} else if ("geo_point".equals(esType)) {
|
||||
if (!(val instanceof String)) {
|
||||
logger.error("es type is geo_point, but source type is not String");
|
||||
return val;
|
||||
}
|
||||
|
||||
if (!((String) val).contains(",")) {
|
||||
logger.error("es type is geo_point, source value not contains ',' separator");
|
||||
return val;
|
||||
}
|
||||
|
||||
String[] point = ((String) val).split(",");
|
||||
Map<String, Double> location = new HashMap<>();
|
||||
location.put("lat", Double.valueOf(point[0].trim()));
|
||||
location.put("lon", Double.valueOf(point[1].trim()));
|
||||
return location;
|
||||
} else if ("array".equals(esType)) {
|
||||
if ("".equals(val.toString().trim())) {
|
||||
res = new ArrayList<>();
|
||||
} else {
|
||||
String value = val.toString();
|
||||
String separator = ",";
|
||||
if (!value.contains(",")) {
|
||||
if (value.contains(";")) {
|
||||
separator = ";";
|
||||
} else if (value.contains("|")) {
|
||||
separator = "\\|";
|
||||
} else if (value.contains("-")) {
|
||||
separator = "-";
|
||||
}
|
||||
}
|
||||
String[] values = value.split(separator);
|
||||
return Arrays.asList(values);
|
||||
}
|
||||
} else if ("object".equals(esType)) {
|
||||
if ("".equals(val.toString().trim())) {
|
||||
res = new HashMap<>();
|
||||
} else {
|
||||
res = JSON.parseObject(val.toString(), Map.class);
|
||||
}
|
||||
} else {
|
||||
// 其他类全以字符串处理
|
||||
res = val.toString();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Blob转byte[]
|
||||
*/
|
||||
private static byte[] blobToBytes(Blob blob) {
|
||||
try (InputStream is = blob.getBinaryStream()) {
|
||||
byte[] b = new byte[(int) blob.length()];
|
||||
if (is.read(b) != -1) {
|
||||
return b;
|
||||
} else {
|
||||
return new byte[0];
|
||||
}
|
||||
} catch (IOException | SQLException e) {
|
||||
logger.error(e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼接主键条件
|
||||
*
|
||||
* @param mapping
|
||||
* @param data
|
||||
* @return
|
||||
*/
|
||||
public static String pkConditionSql(ESMapping mapping, Map<String, Object> data) {
|
||||
Set<ColumnItem> idColumns = new LinkedHashSet<>();
|
||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||
|
||||
TableItem mainTable = schemaItem.getMainTable();
|
||||
|
||||
for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) {
|
||||
if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null)
|
||||
|| (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) {
|
||||
idColumns.add(idColumnItem);
|
||||
}
|
||||
}
|
||||
|
||||
if (idColumns.isEmpty()) {
|
||||
throw new RuntimeException("Not found primary key field in main table");
|
||||
}
|
||||
|
||||
// 拼接condition
|
||||
StringBuilder condition = new StringBuilder(" ");
|
||||
for (ColumnItem idColumn : idColumns) {
|
||||
Object idVal = data.get(idColumn.getColumnName());
|
||||
if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append(".");
|
||||
condition.append(idColumn.getColumnName()).append("=");
|
||||
if (idVal instanceof String) {
|
||||
condition.append("'").append(idVal).append("' AND ");
|
||||
} else {
|
||||
condition.append(idVal).append(" AND ");
|
||||
}
|
||||
}
|
||||
|
||||
if (condition.toString().endsWith("AND ")) {
|
||||
int len2 = condition.length();
|
||||
condition.delete(len2 - 4, len2);
|
||||
}
|
||||
return condition.toString();
|
||||
}
|
||||
|
||||
public static String appendCondition(String sql, String condition) {
|
||||
return sql + " WHERE " + condition + " ";
|
||||
}
|
||||
|
||||
public static void appendCondition(StringBuilder sql, Object value, String owner, String columnName) {
|
||||
if (value instanceof String) {
|
||||
sql.append(owner).append(".").append(columnName).append("='").append(value).append("' AND ");
|
||||
} else {
|
||||
sql.append(owner).append(".").append(columnName).append("=").append(value).append(" AND ");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,523 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.support;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.elasticsearch.action.bulk.BulkItemResponse;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.rest.RestStatus;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.*;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||
|
||||
/**
|
||||
* ES 操作模板
|
||||
*
|
||||
* @author rewerma 2018-11-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ESTemplate {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class);
|
||||
|
||||
private static final int MAX_BATCH_SIZE = 1000;
|
||||
|
||||
private ESConnection esConnection;
|
||||
|
||||
private ESBulkRequest esBulkRequest;
|
||||
|
||||
public ESTemplate(ESConnection esConnection){
|
||||
this.esConnection = esConnection;
|
||||
this.esBulkRequest = this.esConnection.new ESBulkRequest();
|
||||
}
|
||||
|
||||
public ESBulkRequest getBulk() {
|
||||
return esBulkRequest;
|
||||
}
|
||||
|
||||
public void resetBulkRequestBuilder() {
|
||||
this.esBulkRequest.resetBulk();
|
||||
}
|
||||
|
||||
/**
|
||||
* 插入数据
|
||||
*
|
||||
* @param mapping 配置对象
|
||||
* @param pkVal 主键值
|
||||
* @param esFieldData 数据Map
|
||||
*/
|
||||
public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
|
||||
if (mapping.get_id() != null) {
|
||||
String parentVal = (String) esFieldData.remove("$parent_routing");
|
||||
if (mapping.isUpsert()) {
|
||||
ESUpdateRequest updateRequest = esConnection.new ESUpdateRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
|
||||
if (StringUtils.isNotEmpty(parentVal)) {
|
||||
updateRequest.setRouting(parentVal);
|
||||
}
|
||||
getBulk().add(updateRequest);
|
||||
} else {
|
||||
ESIndexRequest indexRequest = esConnection.new ESIndexRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
pkVal.toString()).setSource(esFieldData);
|
||||
if (StringUtils.isNotEmpty(parentVal)) {
|
||||
indexRequest.setRouting(parentVal);
|
||||
}
|
||||
getBulk().add(indexRequest);
|
||||
}
|
||||
commitBulk();
|
||||
} else {
|
||||
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
|
||||
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
|
||||
SearchResponse response = esSearchRequest.getResponse();
|
||||
|
||||
for (SearchHit hit : response.getHits()) {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
hit.getId()).setDoc(esFieldData);
|
||||
getBulk().add(esUpdateRequest);
|
||||
commitBulk();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据主键更新数据
|
||||
*
|
||||
* @param mapping 配置对象
|
||||
* @param pkVal 主键值
|
||||
* @param esFieldData 数据Map
|
||||
*/
|
||||
public void update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
|
||||
Map<String, Object> esFieldDataTmp = new LinkedHashMap<>(esFieldData.size());
|
||||
esFieldData.forEach((k, v) -> esFieldDataTmp.put(Util.cleanColumn(k), v));
|
||||
append4Update(mapping, pkVal, esFieldDataTmp);
|
||||
commitBulk();
|
||||
}
|
||||
|
||||
/**
|
||||
* update by query
|
||||
*
|
||||
* @param config 配置对象
|
||||
* @param paramsTmp sql查询条件
|
||||
* @param esFieldData 数据Map
|
||||
*/
|
||||
public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
|
||||
if (paramsTmp.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
ESMapping mapping = config.getEsMapping();
|
||||
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
|
||||
paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
|
||||
|
||||
// 查询sql批量更新
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
|
||||
StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
|
||||
List<Object> values = new ArrayList<>();
|
||||
paramsTmp.forEach((fieldName, value) -> {
|
||||
sql.append("_v.").append(fieldName).append("=? AND ");
|
||||
values.add(value);
|
||||
});
|
||||
// TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
|
||||
int len = sql.length();
|
||||
sql.delete(len - 4, len);
|
||||
Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
|
||||
int count = 0;
|
||||
try {
|
||||
while (rs.next()) {
|
||||
Object idVal = getIdValFromRS(mapping, rs);
|
||||
append4Update(mapping, idVal, esFieldData);
|
||||
commitBulk();
|
||||
count++;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return count;
|
||||
});
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Update ES by query affected {} records", syncCount);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过主键删除数据
|
||||
*
|
||||
* @param mapping 配置对象
|
||||
* @param pkVal 主键值
|
||||
* @param esFieldData 数据Map
|
||||
*/
|
||||
public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
|
||||
if (mapping.get_id() != null) {
|
||||
ESDeleteRequest esDeleteRequest = this.esConnection.new ESDeleteRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
pkVal.toString());
|
||||
getBulk().add(esDeleteRequest);
|
||||
commitBulk();
|
||||
} else {
|
||||
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
|
||||
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
|
||||
SearchResponse response = esSearchRequest.getResponse();
|
||||
for (SearchHit hit : response.getHits()) {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
hit.getId()).setDoc(esFieldData);
|
||||
getBulk().add(esUpdateRequest);
|
||||
commitBulk();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交批次
|
||||
*/
|
||||
public void commit() {
|
||||
if (getBulk().numberOfActions() > 0) {
|
||||
BulkResponse response = getBulk().bulk();
|
||||
if (response.hasFailures()) {
|
||||
for (BulkItemResponse itemResponse : response.getItems()) {
|
||||
if (!itemResponse.isFailed()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
|
||||
logger.error(itemResponse.getFailureMessage());
|
||||
} else {
|
||||
throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
resetBulkRequestBuilder();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 如果大于批量数则提交批次
|
||||
*/
|
||||
private void commitBulk() {
|
||||
if (getBulk().numberOfActions() >= MAX_BATCH_SIZE) {
|
||||
commit();
|
||||
}
|
||||
}
|
||||
|
||||
private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
|
||||
if (mapping.get_id() != null) {
|
||||
String parentVal = (String) esFieldData.remove("$parent_routing");
|
||||
if (mapping.isUpsert()) {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
|
||||
if (StringUtils.isNotEmpty(parentVal)) {
|
||||
esUpdateRequest.setRouting(parentVal);
|
||||
}
|
||||
getBulk().add(esUpdateRequest);
|
||||
} else {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
pkVal.toString()).setDoc(esFieldData);
|
||||
if (StringUtils.isNotEmpty(parentVal)) {
|
||||
esUpdateRequest.setRouting(parentVal);
|
||||
}
|
||||
getBulk().add(esUpdateRequest);
|
||||
}
|
||||
} else {
|
||||
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
|
||||
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
|
||||
SearchResponse response = esSearchRequest.getResponse();
|
||||
for (SearchHit hit : response.getHits()) {
|
||||
ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
|
||||
mapping.get_type(),
|
||||
hit.getId()).setDoc(esFieldData);
|
||||
getBulk().add(esUpdateRequest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName,
|
||||
String columnName) throws SQLException {
|
||||
fieldName = Util.cleanColumn(fieldName);
|
||||
columnName = Util.cleanColumn(columnName);
|
||||
String esType = getEsType(mapping, fieldName);
|
||||
|
||||
Object value = resultSet.getObject(columnName);
|
||||
if (value instanceof Boolean) {
|
||||
if (!"boolean".equals(esType)) {
|
||||
value = resultSet.getByte(columnName);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果是对象类型
|
||||
if (mapping.getObjFields().containsKey(fieldName)) {
|
||||
return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
|
||||
} else {
|
||||
return ESSyncUtil.typeConvert(value, esType);
|
||||
}
|
||||
}
|
||||
|
||||
public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,
|
||||
Map<String, Object> esFieldData) throws SQLException {
|
||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||
Object resultIdVal = null;
|
||||
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
|
||||
|
||||
if (fieldItem.getFieldName().equals(idFieldName)) {
|
||||
resultIdVal = value;
|
||||
}
|
||||
|
||||
if (!fieldItem.getFieldName().equals(mapping.get_id())
|
||||
&& !mapping.getSkips().contains(fieldItem.getFieldName())) {
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加父子文档关联信息
|
||||
putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
|
||||
|
||||
return resultIdVal;
|
||||
}
|
||||
|
||||
public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
|
||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||
Object resultIdVal = null;
|
||||
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
|
||||
|
||||
if (fieldItem.getFieldName().equals(idFieldName)) {
|
||||
resultIdVal = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return resultIdVal;
|
||||
}
|
||||
|
||||
public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
|
||||
Map<String, Object> esFieldData) throws SQLException {
|
||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||
Object resultIdVal = null;
|
||||
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
if (fieldItem.getFieldName().equals(idFieldName)) {
|
||||
resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
|
||||
}
|
||||
|
||||
for (ColumnItem columnItem : fieldItem.getColumnItems()) {
|
||||
if (dmlOld.containsKey(columnItem.getColumnName())
|
||||
&& !mapping.getSkips().contains(fieldItem.getFieldName())) {
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
|
||||
getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 添加父子文档关联信息
|
||||
putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
|
||||
|
||||
return resultIdVal;
|
||||
}
|
||||
|
||||
public Object getValFromData(ESMapping mapping, Map<String, Object> dmlData, String fieldName, String columnName) {
|
||||
String esType = getEsType(mapping, fieldName);
|
||||
Object value = dmlData.get(columnName);
|
||||
if (value instanceof Byte) {
|
||||
if ("boolean".equals(esType)) {
|
||||
value = ((Byte) value).intValue() != 0;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果是对象类型
|
||||
if (mapping.getObjFields().containsKey(fieldName)) {
|
||||
return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
|
||||
} else {
|
||||
return ESSyncUtil.typeConvert(value, esType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将dml的data转换为es的data
|
||||
*
|
||||
* @param mapping 配置mapping
|
||||
* @param dmlData dml data
|
||||
* @param esFieldData es data
|
||||
* @return 返回 id 值
|
||||
*/
|
||||
public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
|
||||
Map<String, Object> esFieldData) {
|
||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||
Object resultIdVal = null;
|
||||
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
|
||||
Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
|
||||
|
||||
if (fieldItem.getFieldName().equals(idFieldName)) {
|
||||
resultIdVal = value;
|
||||
}
|
||||
|
||||
if (!fieldItem.getFieldName().equals(mapping.get_id())
|
||||
&& !mapping.getSkips().contains(fieldItem.getFieldName())) {
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加父子文档关联信息
|
||||
putRelationData(mapping, schemaItem, dmlData, esFieldData);
|
||||
return resultIdVal;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将dml的data, old转换为es的data
|
||||
*
|
||||
* @param mapping 配置mapping
|
||||
* @param owner 所属表
|
||||
* @param dmlData dml data
|
||||
* @param esFieldData es data
|
||||
* @return 返回 id 值
|
||||
*/
|
||||
public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
|
||||
Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
|
||||
SchemaItem schemaItem = mapping.getSchemaItem();
|
||||
String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
|
||||
Object resultIdVal = null;
|
||||
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
|
||||
ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
|
||||
if (!columnItem.getOwner().equals(owner)) {
|
||||
continue;
|
||||
}
|
||||
String columnName = columnItem.getColumnName();
|
||||
|
||||
if (fieldItem.getFieldName().equals(idFieldName)) {
|
||||
resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
|
||||
}
|
||||
|
||||
if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
|
||||
esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
|
||||
getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
|
||||
}
|
||||
}
|
||||
|
||||
// 添加父子文档关联信息
|
||||
putRelationData(mapping, schemaItem, dmlOld, esFieldData);
|
||||
return resultIdVal;
|
||||
}
|
||||
|
||||
private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet,
|
||||
Map<String, Object> esFieldData) {
|
||||
// 添加父子文档关联信息
|
||||
if (!mapping.getRelations().isEmpty()) {
|
||||
mapping.getRelations().forEach((relationField, relationMapping) -> {
|
||||
Map<String, Object> relations = new HashMap<>();
|
||||
relations.put("name", relationMapping.getName());
|
||||
if (StringUtils.isNotEmpty(relationMapping.getParent())) {
|
||||
FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
|
||||
Object parentVal;
|
||||
try {
|
||||
parentVal = getValFromRS(mapping,
|
||||
resultSet,
|
||||
parentFieldItem.getFieldName(),
|
||||
parentFieldItem.getFieldName());
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
if (parentVal != null) {
|
||||
relations.put("parent", parentVal.toString());
|
||||
esFieldData.put("$parent_routing", parentVal.toString());
|
||||
|
||||
}
|
||||
}
|
||||
esFieldData.put(relationField, relations);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map<String, Object> dmlData,
|
||||
Map<String, Object> esFieldData) {
|
||||
// 添加父子文档关联信息
|
||||
if (!mapping.getRelations().isEmpty()) {
|
||||
mapping.getRelations().forEach((relationField, relationMapping) -> {
|
||||
Map<String, Object> relations = new HashMap<>();
|
||||
relations.put("name", relationMapping.getName());
|
||||
if (StringUtils.isNotEmpty(relationMapping.getParent())) {
|
||||
FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
|
||||
String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName();
|
||||
Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName);
|
||||
if (parentVal != null) {
|
||||
relations.put("parent", parentVal.toString());
|
||||
esFieldData.put("$parent_routing", parentVal.toString());
|
||||
|
||||
}
|
||||
}
|
||||
esFieldData.put(relationField, relations);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* es 字段类型本地缓存
|
||||
*/
|
||||
private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 获取es mapping中的属性类型
|
||||
*
|
||||
* @param mapping mapping配置
|
||||
* @param fieldName 属性名
|
||||
* @return 类型
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private String getEsType(ESMapping mapping, String fieldName) {
|
||||
String key = mapping.get_index() + "-" + mapping.get_type();
|
||||
Map<String, String> fieldType = esFieldTypes.get(key);
|
||||
if (fieldType != null) {
|
||||
return fieldType.get(fieldName);
|
||||
} else {
|
||||
MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index(), mapping.get_type());
|
||||
|
||||
if (mappingMetaData == null) {
|
||||
throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
|
||||
}
|
||||
|
||||
fieldType = new LinkedHashMap<>();
|
||||
|
||||
Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
|
||||
Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
|
||||
for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
|
||||
Map<String, Object> value = (Map<String, Object>) entry.getValue();
|
||||
if (value.containsKey("properties")) {
|
||||
fieldType.put(entry.getKey(), "object");
|
||||
} else {
|
||||
fieldType.put(entry.getKey(), (String) value.get("type"));
|
||||
}
|
||||
}
|
||||
esFieldTypes.put(key, fieldType);
|
||||
|
||||
return fieldType.get(fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package org.elasticsearch.client;
|
||||
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.common.Strings;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* RequestConverters扩展
|
||||
*
|
||||
* @author rewerma 2019-08-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class RequestConvertersExt {
|
||||
|
||||
/**
|
||||
* 修改 getMappings 去掉request参数
|
||||
*
|
||||
* @param getMappingsRequest
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOException {
|
||||
String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices();
|
||||
String[] types = getMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.types();
|
||||
|
||||
return new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping", types));
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package org.elasticsearch.client;
|
||||
|
||||
import static java.util.Collections.emptySet;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
|
||||
/**
|
||||
* RestHighLevelClient扩展
|
||||
*
|
||||
* @author rewerma 2019-08-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class RestHighLevelClientExt {
|
||||
|
||||
public static GetMappingsResponse getMapping(RestHighLevelClient restHighLevelClient,
|
||||
GetMappingsRequest getMappingsRequest,
|
||||
RequestOptions options) throws IOException {
|
||||
return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest,
|
||||
RequestConvertersExt::getMappings,
|
||||
options,
|
||||
GetMappingsResponse::fromXContent,
|
||||
emptySet());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
es=com.alibaba.otter.canal.client.adapter.es.ESAdapter
|
||||
@@ -1,21 +0,0 @@
|
||||
dataSourceKey: defaultDS
|
||||
destination: example
|
||||
groupId: g1
|
||||
esMapping:
|
||||
_index: customer
|
||||
_type: _doc
|
||||
_id: _id
|
||||
relations:
|
||||
customer_order:
|
||||
name: order
|
||||
parent: customer_id
|
||||
sql: "select concat('oid_', t.id) as _id,
|
||||
t.customer_id,
|
||||
t.id as order_id,
|
||||
t.serial_code as order_serial,
|
||||
t.c_time as order_time
|
||||
from biz_order t"
|
||||
skips:
|
||||
- customer_id
|
||||
etlCondition: "where t.c_time>={}"
|
||||
commitBatch: 3000
|
||||
@@ -1,47 +0,0 @@
|
||||
dataSourceKey: defaultDS
|
||||
destination: example
|
||||
groupId: g1
|
||||
esMapping:
|
||||
_index: customer
|
||||
_type: _doc
|
||||
_id: id
|
||||
relations:
|
||||
customer_order:
|
||||
name: customer
|
||||
sql: "select t.id, t.name, t.email from customer t"
|
||||
etlCondition: "where t.c_time>={}"
|
||||
commitBatch: 3000
|
||||
|
||||
|
||||
#{
|
||||
# "mappings":{
|
||||
# "_doc":{
|
||||
# "properties":{
|
||||
# "id": {
|
||||
# "type": "long"
|
||||
# },
|
||||
# "name": {
|
||||
# "type": "text"
|
||||
# },
|
||||
# "email": {
|
||||
# "type": "text"
|
||||
# },
|
||||
# "order_id": {
|
||||
# "type": "long"
|
||||
# },
|
||||
# "order_serial": {
|
||||
# "type": "text"
|
||||
# },
|
||||
# "order_time": {
|
||||
# "type": "date"
|
||||
# },
|
||||
# "customer_order":{
|
||||
# "type":"join",
|
||||
# "relations":{
|
||||
# "customer":"order"
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
#}
|
||||
@@ -1,16 +0,0 @@
|
||||
dataSourceKey: defaultDS
|
||||
destination: example
|
||||
groupId: g1
|
||||
esMapping:
|
||||
_index: mytest_user
|
||||
_type: _doc
|
||||
_id: _id
|
||||
upsert: true
|
||||
# pk: id
|
||||
sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,
|
||||
a.c_time as _c_time from user a
|
||||
left join role b on b.id=a.role_id"
|
||||
# objFields:
|
||||
# _labels: array:;
|
||||
etlCondition: "where a.c_time>={}"
|
||||
commitBatch: 3000
|
||||
@@ -1,40 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
@Ignore
|
||||
public class ConfigLoadTest {
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
// AdapterConfigs.put("es", "mytest_user.yml");
|
||||
// 加载数据源连接池
|
||||
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoad() {
|
||||
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load(null);
|
||||
ESSyncConfig config = configMap.get("mytest_user.yml");
|
||||
config.validate();
|
||||
Assert.assertNotNull(config);
|
||||
Assert.assertEquals("defaultDS", config.getDataSourceKey());
|
||||
ESSyncConfig.ESMapping esMapping = config.getEsMapping();
|
||||
Assert.assertEquals("mytest_user", esMapping.get_index());
|
||||
Assert.assertEquals("_doc", esMapping.get_type());
|
||||
Assert.assertEquals("id", esMapping.get_id());
|
||||
Assert.assertNotNull(esMapping.getSql());
|
||||
|
||||
// Map<String, List<ESSyncConfig>> dbTableEsSyncConfig =
|
||||
// ESSyncConfigLoader.getDbTableEsSyncConfig();
|
||||
// Assert.assertFalse(dbTableEsSyncConfig.isEmpty());
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.elasticsearch.action.bulk.BulkItemResponse;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.TransportAddress;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.rest.RestStatus;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.transport.client.PreBuiltTransportClient;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
@Ignore
|
||||
public class ESTest {
|
||||
|
||||
private TransportClient transportClient;
|
||||
|
||||
@Before
|
||||
public void init() throws UnknownHostException {
|
||||
Settings.Builder settingBuilder = Settings.builder();
|
||||
settingBuilder.put("cluster.name", TestConstant.clusterName);
|
||||
Settings settings = settingBuilder.build();
|
||||
transportClient = new PreBuiltTransportClient(settings);
|
||||
String[] hostArray = TestConstant.esHosts.split(",");
|
||||
for (String host : hostArray) {
|
||||
int i = host.indexOf(":");
|
||||
transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
|
||||
Integer.parseInt(host.substring(i + 1))));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test01() {
|
||||
SearchResponse response = transportClient.prepareSearch("test")
|
||||
.setTypes("osm")
|
||||
.setQuery(QueryBuilders.termQuery("_id", "1"))
|
||||
.setSize(10000)
|
||||
.get();
|
||||
for (SearchHit hit : response.getHits()) {
|
||||
System.out.println(hit.getSourceAsMap().get("data").getClass());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test02() {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
esFieldData.put("userId", 2L);
|
||||
esFieldData.put("eventId", 4L);
|
||||
esFieldData.put("eventName", "网络异常");
|
||||
esFieldData.put("description", "第四个事件信息");
|
||||
|
||||
Map<String, Object> relations = new LinkedHashMap<>();
|
||||
esFieldData.put("user_event", relations);
|
||||
relations.put("name", "event");
|
||||
relations.put("parent", "2");
|
||||
|
||||
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
|
||||
bulkRequestBuilder
|
||||
.add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData));
|
||||
commit(bulkRequestBuilder);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test03() {
|
||||
Map<String, Object> esFieldData = new LinkedHashMap<>();
|
||||
esFieldData.put("userId", 2L);
|
||||
esFieldData.put("eventName", "网络异常1");
|
||||
|
||||
Map<String, Object> relations = new LinkedHashMap<>();
|
||||
esFieldData.put("user_event", relations);
|
||||
relations.put("name", "event");
|
||||
relations.put("parent", "2");
|
||||
|
||||
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
|
||||
bulkRequestBuilder.add(transportClient.prepareUpdate("test", "osm", "2_4").setRouting("2").setDoc(esFieldData));
|
||||
commit(bulkRequestBuilder);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test04() {
|
||||
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
|
||||
bulkRequestBuilder.add(transportClient.prepareDelete("test", "osm", "2_4"));
|
||||
commit(bulkRequestBuilder);
|
||||
}
|
||||
|
||||
private void commit(BulkRequestBuilder bulkRequestBuilder) {
|
||||
if (bulkRequestBuilder.numberOfActions() > 0) {
|
||||
BulkResponse response = bulkRequestBuilder.execute().actionGet();
|
||||
if (response.hasFailures()) {
|
||||
for (BulkItemResponse itemResponse : response.getItems()) {
|
||||
if (!itemResponse.isFailed()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
|
||||
System.out.println(itemResponse.getFailureMessage());
|
||||
} else {
|
||||
System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@After
|
||||
public void after() {
|
||||
transportClient.close();
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
|
||||
|
||||
public class SqlParseTest {
|
||||
|
||||
@Test
|
||||
public void parseTest() {
|
||||
String sql = "select a.id, CASE WHEN a.id <= 500 THEN '1' else '2' end as id2, "
|
||||
+ "concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a "
|
||||
+ "left join role b on a.role_id=b.id "
|
||||
+ "left join (select user_id, group_concat(label,',') as labels from user_label "
|
||||
+ "group by user_id) c on c.user_id=a.id";
|
||||
SchemaItem schemaItem = SqlParser.parse(sql);
|
||||
|
||||
// 通过表名找 TableItem
|
||||
List<TableItem> tableItems = schemaItem.getTableItemAliases().get("user_label".toLowerCase());
|
||||
tableItems.forEach(tableItem -> Assert.assertEquals("c", tableItem.getAlias()));
|
||||
|
||||
TableItem tableItem = tableItems.get(0);
|
||||
Assert.assertFalse(tableItem.isMain());
|
||||
Assert.assertTrue(tableItem.isSubQuery());
|
||||
// 通过字段名找 FieldItem
|
||||
List<FieldItem> fieldItems = schemaItem.getColumnFields().get(tableItem.getAlias() + ".labels".toLowerCase());
|
||||
fieldItems.forEach(
|
||||
fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName()));
|
||||
|
||||
// 获取当前表关联条件字段
|
||||
Map<FieldItem, List<FieldItem>> relationTableFields = tableItem.getRelationTableFields();
|
||||
relationTableFields.keySet()
|
||||
.forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName()));
|
||||
|
||||
// 获取关联字段在select中的对应字段
|
||||
// List<FieldItem> relationSelectFieldItem =
|
||||
// tableItem.getRelationKeyFieldItems();
|
||||
// relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels",
|
||||
// fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName()));
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test;
|
||||
|
||||
import java.sql.SQLException;
|
||||
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
|
||||
public class TestConstant {
|
||||
|
||||
public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
|
||||
public final static String jdbcUser = "root";
|
||||
public final static String jdbcPassword = "121212";
|
||||
|
||||
public final static String esHosts = "127.0.0.1:9300";
|
||||
public final static String clusterName = "elasticsearch";
|
||||
|
||||
public final static DruidDataSource dataSource;
|
||||
|
||||
static {
|
||||
dataSource = new DruidDataSource();
|
||||
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
|
||||
dataSource.setUrl(jdbcUrl);
|
||||
dataSource.setUsername(jdbcUser);
|
||||
dataSource.setPassword(jdbcPassword);
|
||||
dataSource.setInitialSize(1);
|
||||
dataSource.setMinIdle(1);
|
||||
dataSource.setMaxActive(1);
|
||||
dataSource.setMaxWait(60000);
|
||||
dataSource.setTimeBetweenEvictionRunsMillis(60000);
|
||||
dataSource.setMinEvictableIdleTimeMillis(300000);
|
||||
dataSource.setPoolPreparedStatements(false);
|
||||
dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
|
||||
dataSource.setValidationQuery("select 1");
|
||||
try {
|
||||
dataSource.init();
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.test.TestConstant;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||
|
||||
public class Common {
|
||||
|
||||
public static ESAdapter init() {
|
||||
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
|
||||
|
||||
OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
|
||||
outerAdapterConfig.setName("es");
|
||||
outerAdapterConfig.setHosts(TestConstant.esHosts);
|
||||
Map<String, String> properties = new HashMap<>();
|
||||
properties.put("cluster.name", TestConstant.clusterName);
|
||||
outerAdapterConfig.setProperties(properties);
|
||||
|
||||
ESAdapter esAdapter = new ESAdapter();
|
||||
esAdapter.init(outerAdapterConfig, null);
|
||||
return esAdapter;
|
||||
}
|
||||
|
||||
public static void sqlExe(DataSource dataSource, String sql) {
|
||||
Connection conn = null;
|
||||
Statement stmt = null;
|
||||
try {
|
||||
conn = dataSource.getConnection();
|
||||
conn.setAutoCommit(false);
|
||||
stmt = conn.createStatement();
|
||||
stmt.execute(sql);
|
||||
conn.commit();
|
||||
} catch (Exception e) {
|
||||
if (conn != null) {
|
||||
try {
|
||||
conn.rollback();
|
||||
} catch (SQLException e1) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if (stmt != null) {
|
||||
try {
|
||||
stmt.close();
|
||||
} catch (SQLException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
if (conn != null) {
|
||||
try {
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
|
||||
@Ignore
|
||||
public class LabelSyncJoinSub2Test {
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
// AdapterConfigs.put("es", "mytest_user_join_sub2.yml");
|
||||
esAdapter = Common.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* 带函数子查询从表插入
|
||||
*/
|
||||
@Test
|
||||
public void test01() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from label where id=1 or id=2");
|
||||
Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
|
||||
Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("INSERT");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("label");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 2L);
|
||||
data.put("user_id", 1L);
|
||||
data.put("label", "b");
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("b;a_", response.getSource().get("_labels"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 带函数子查询从表更新
|
||||
*/
|
||||
@Test
|
||||
public void test02() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "update label set label='aa' where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("label");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("user_id", 1L);
|
||||
data.put("label", "aa");
|
||||
dml.setData(dataList);
|
||||
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("label", "v");
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 带函数子查询从表删除
|
||||
*/
|
||||
@Test
|
||||
public void test03() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from label where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("DELETE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("label");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("user_id", 1L);
|
||||
data.put("label", "a");
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("b_", response.getSource().get("_labels"));
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
|
||||
@Ignore
|
||||
public class LabelSyncJoinSubTest {
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
// AdapterConfigs.put("es", "mytest_user_join_sub.yml");
|
||||
esAdapter = Common.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* 子查询从表插入
|
||||
*/
|
||||
@Test
|
||||
public void test01() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from label where id=1 or id=2");
|
||||
Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
|
||||
Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("INSERT");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("label");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 2L);
|
||||
data.put("user_id", 1L);
|
||||
data.put("label", "b");
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("b;a", response.getSource().get("_labels"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 子查询从表更新
|
||||
*/
|
||||
@Test
|
||||
public void test02() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "update label set label='aa' where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("label");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("user_id", 1L);
|
||||
data.put("label", "aa");
|
||||
dml.setData(dataList);
|
||||
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("label", "a");
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("b;aa", response.getSource().get("_labels"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 子查询从表删除
|
||||
*/
|
||||
@Test
|
||||
public void test03() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from label where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("DELETE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("label");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("user_id", 1L);
|
||||
data.put("label", "a");
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("b", response.getSource().get("_labels"));
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
|
||||
@Ignore
|
||||
public class RoleSyncJoinOne2Test {
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
// AdapterConfigs.put("es", "mytest_user_join_one2.yml");
|
||||
esAdapter = Common.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* 带函数非子查询从表插入
|
||||
*/
|
||||
@Test
|
||||
public void test01() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from role where id=1");
|
||||
Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("INSERT");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("role");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("role_name", "admin");
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("admin_", response.getSource().get("_role_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 带函数非子查询从表更新
|
||||
*/
|
||||
@Test
|
||||
public void test02() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "update role set role_name='admin3' where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("role");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("role_name", "admin3");
|
||||
dml.setData(dataList);
|
||||
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("role_name", "admin");
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("admin3_", response.getSource().get("_role_name"));
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
@Ignore
|
||||
public class RoleSyncJoinOneTest {
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
// AdapterConfigs.put("es", "mytest_user_join_one.yml");
|
||||
esAdapter = Common.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* 非子查询从表插入
|
||||
*/
|
||||
@Test
|
||||
public void test01() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from role where id=1");
|
||||
Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("INSERT");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("role");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("role_name", "admin");
|
||||
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
|
||||
Assert.assertEquals("admin", response.getSource().get("_role_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 非子查询从表更新
|
||||
*/
|
||||
@Test
|
||||
public void test02() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "update role set role_name='admin2' where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("role");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("role_name", "admin2");
|
||||
dml.setData(dataList);
|
||||
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("role_name", "admin");
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
|
||||
Assert.assertEquals("admin2", response.getSource().get("_role_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 主表更新外键值
|
||||
*/
|
||||
@Test
|
||||
public void test03() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from role where id=2");
|
||||
Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')");
|
||||
Common.sqlExe(ds, "update user set role_id=2 where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("user");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("role_id", 2L);
|
||||
dml.setData(dataList);
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("role_id", 1L);
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
|
||||
Assert.assertEquals("operator", response.getSource().get("_role_name"));
|
||||
|
||||
Common.sqlExe(ds, "update user set role_id=1 where id=1");
|
||||
|
||||
Dml dml2 = new Dml();
|
||||
dml2.setDestination("example");
|
||||
dml2.setTs(new Date().getTime());
|
||||
dml2.setType("UPDATE");
|
||||
dml2.setDatabase("mytest");
|
||||
dml2.setTable("user");
|
||||
List<Map<String, Object>> dataList2 = new ArrayList<>();
|
||||
Map<String, Object> data2 = new LinkedHashMap<>();
|
||||
dataList2.add(data2);
|
||||
data2.put("id", 1L);
|
||||
data2.put("role_id", 1L);
|
||||
dml2.setData(dataList2);
|
||||
List<Map<String, Object>> oldList2 = new ArrayList<>();
|
||||
Map<String, Object> old2 = new LinkedHashMap<>();
|
||||
oldList2.add(old2);
|
||||
old2.put("role_id", 2L);
|
||||
dml2.setOld(oldList2);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2);
|
||||
|
||||
GetResponse response2 = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
|
||||
Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 非子查询从表删除
|
||||
*/
|
||||
@Test
|
||||
public void test04() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from role where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("DELETE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("role");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("role_name", "admin");
|
||||
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
|
||||
Assert.assertNull(response.getSource().get("_role_name"));
|
||||
}
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
|
||||
@Ignore
|
||||
public class UserSyncJoinOneTest {
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
// AdapterConfigs.put("es", "mytest_user_join_one.yml");
|
||||
esAdapter = Common.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* 主表带函数插入
|
||||
*/
|
||||
@Test
|
||||
public void test01() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "delete from user where id=1");
|
||||
Common.sqlExe(ds, "insert into user (id,name,role_id) values (1,'Eric',1)");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("INSERT");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("user");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("name", "Eric");
|
||||
data.put("role_id", 1L);
|
||||
data.put("c_time", new Date());
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("Eric_", response.getSource().get("_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 主表带函数更新
|
||||
*/
|
||||
@Test
|
||||
public void test02() {
|
||||
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
|
||||
Common.sqlExe(ds, "update user set name='Eric2' where id=1");
|
||||
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("user");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("name", "Eric2");
|
||||
dml.setData(dataList);
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("name", "Eric");
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("Eric2_", response.getSource().get("_name"));
|
||||
}
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package com.alibaba.otter.canal.client.adapter.es.test.sync;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
|
||||
@Ignore
|
||||
public class UserSyncSingleTest {
|
||||
|
||||
private ESAdapter esAdapter;
|
||||
|
||||
@Before
|
||||
public void init() {
|
||||
// AdapterConfigs.put("es", "mytest_user_single.yml");
|
||||
esAdapter = Common.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* 单表插入
|
||||
*/
|
||||
@Test
|
||||
public void test01() {
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("INSERT");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("user");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("name", "Eric");
|
||||
data.put("role_id", 1L);
|
||||
data.put("c_time", new Date());
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("Eric", response.getSource().get("_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 单表更新
|
||||
*/
|
||||
@Test
|
||||
public void test02() {
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("UPDATE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("user");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("name", "Eric2");
|
||||
dml.setData(dataList);
|
||||
List<Map<String, Object>> oldList = new ArrayList<>();
|
||||
Map<String, Object> old = new LinkedHashMap<>();
|
||||
oldList.add(old);
|
||||
old.put("name", "Eric");
|
||||
dml.setOld(oldList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertEquals("Eric2", response.getSource().get("_name"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 单表删除
|
||||
*/
|
||||
@Test
|
||||
public void test03() {
|
||||
Dml dml = new Dml();
|
||||
dml.setDestination("example");
|
||||
dml.setTs(new Date().getTime());
|
||||
dml.setType("DELETE");
|
||||
dml.setDatabase("mytest");
|
||||
dml.setTable("user");
|
||||
List<Map<String, Object>> dataList = new ArrayList<>();
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
dataList.add(data);
|
||||
data.put("id", 1L);
|
||||
data.put("name", "Eric");
|
||||
data.put("role_id", 1L);
|
||||
data.put("c_time", new Date());
|
||||
dml.setData(dataList);
|
||||
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
|
||||
|
||||
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
|
||||
|
||||
GetResponse response = esAdapter.getEsConnection()
|
||||
.getTransportClient()
|
||||
.prepareGet("mytest_user", "_doc", "1")
|
||||
.get();
|
||||
Assert.assertNull(response.getSource());
|
||||
}
|
||||
|
||||
// @After
|
||||
// public void after() {
|
||||
// esAdapter.destroy();
|
||||
// DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
|
||||
// }
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
-- ----------------------------
|
||||
-- Table structure for label
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `label`;
|
||||
CREATE TABLE `label` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`user_id` bigint(20) NOT NULL,
|
||||
`label` varchar(30) NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for role
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `role`;
|
||||
CREATE TABLE `role` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`role_name` varchar(30) NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for user
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS `user`;
|
||||
CREATE TABLE `user` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(30) NOT NULL,
|
||||
`c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`role_id` bigint(20) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
insert into user (id,name,role_id) values (1,'Eric',1);
|
||||
insert into role (id,role_name) values (1,'admin');
|
||||
insert into role (id,role_name) values (2,'operator');
|
||||
insert into label (id,user_id,label) values (1,1,'a');
|
||||
insert into label (id,user_id,label) values (2,1,'b');
|
||||
commit;
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"_doc": {
|
||||
"properties": {
|
||||
"_name": {
|
||||
"type": "text"
|
||||
},
|
||||
"_role_id": {
|
||||
"type": "long"
|
||||
},
|
||||
"_role_name": {
|
||||
"type": "text"
|
||||
},
|
||||
"_labels": {
|
||||
"type": "text"
|
||||
},
|
||||
"_c_time": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
dataSourceKey: defaultDS
|
||||
destination: example
|
||||
esMapping:
|
||||
_index: mytest_user
|
||||
_type: _doc
|
||||
_id: _id
|
||||
sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a"
|
||||
commitBatch: 3000
|
||||
@@ -1,13 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration status="WARN">
|
||||
<Appenders>
|
||||
<Console name="Console" target="SYSTEM_OUT">
|
||||
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
|
||||
</Console>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<Root level="ERROR">
|
||||
<AppenderRef ref="Console"/>
|
||||
</Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||
@@ -1,13 +0,0 @@
|
||||
<configuration scan="true" scanPeriod=" 5 seconds">
|
||||
<jmxConfigurator />
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<root level="TRACE">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -20,7 +20,6 @@
|
||||
<module>common</module>
|
||||
<module>logger</module>
|
||||
<module>hbase</module>
|
||||
<module>elasticsearch</module>
|
||||
<module>launcher</module>
|
||||
<module>rdb</module>
|
||||
<module>es6x</module>
|
||||
|
||||
Reference in New Issue
Block a user