From f45c0206da5fc8882a34b7efcce0d8975612ac1d Mon Sep 17 00:00:00 2001 From: rewerma Date: Wed, 25 Sep 2019 18:51:34 +0800 Subject: [PATCH] Add elasticsearch 7.x version adapter support (#2246) * init es data sync upgrade * Add elasticsearch 7.x version adapter support --- client-adapter/es-core/pom.xml | 49 + .../client/adapter/es/core/ESAdapter.java | 177 ++++ .../adapter/es/core/config/ESSyncConfig.java | 251 +++++ .../es/core/config/ESSyncConfigLoader.java | 47 + .../adapter/es/core/config/SchemaItem.java | 439 +++++++++ .../adapter/es/core/config/SqlParser.java | 276 ++++++ .../es/core/monitor/ESConfigMonitor.java | 146 +++ .../es/core/service/ESSyncService.java | 875 ++++++++++++++++++ .../es/core/support/ESBulkRequest.java | 43 + .../adapter/es/core/support/ESSyncUtil.java | 303 ++++++ .../adapter/es/core/support/ESTemplate.java | 69 ++ client-adapter/es6x/pom.xml | 101 ++ .../client/adapter/es6x/ES6xAdapter.java | 123 +++ .../client/adapter/es6x/etl/ESEtlService.java | 202 ++++ .../adapter/es6x/support/ES6xTemplate.java | 468 ++++++++++ .../adapter/es6x/support/ESConnection.java | 505 ++++++++++ .../client/RequestConvertersExt.java | 30 + .../client/RestHighLevelClientExt.java | 29 + ...ba.otter.canal.client.adapter.OuterAdapter | 1 + .../es6x/src/main/resources/es6/biz_order.yml | 21 + .../es6x/src/main/resources/es6/customer.yml | 47 + .../src/main/resources/es6/mytest_user.yml | 16 + .../adapter/es6x/test/ConfigLoadTest.java | 41 + .../client/adapter/es6x/test/ESTest.java | 120 +++ .../adapter/es6x/test/SqlParseTest.java | 48 + .../adapter/es6x/test/TestConstant.java | 40 + .../client/adapter/es6x/test/sync/Common.java | 68 ++ .../es6x/test/sync/LabelSyncJoinSub2Test.java | 141 +++ .../es6x/test/sync/LabelSyncJoinSubTest.java | 141 +++ .../es6x/test/sync/RoleSyncJoinOne2Test.java | 103 +++ .../es6x/test/sync/RoleSyncJoinOneTest.java | 208 +++++ .../es6x/test/sync/UserSyncJoinOneTest.java | 104 +++ .../es6x/test/sync/UserSyncSingleTest.java | 133 +++ .../adapter/es6x/test/sync/db_schema.sql | 39 + .../adapter/es6x/test/sync/es_mapping.json | 21 + .../resources/es6/mytest_user_single.yml_ | 8 + .../es6x/src/test/resources/log4j2-test.xml | 13 + .../es6x/src/test/resources/logback-test.xml | 13 + client-adapter/es7x/pom.xml | 100 ++ .../client/adapter/es7x/ES7xAdapter.java | 123 +++ .../client/adapter/es7x/etl/ESEtlService.java | 202 ++++ .../adapter/es7x/support/ES7xTemplate.java | 462 +++++++++ .../adapter/es7x/support/ESConnection.java | 504 ++++++++++ ...ba.otter.canal.client.adapter.OuterAdapter | 1 + .../es7x/src/main/resources/es7/biz_order.yml | 20 + .../es7x/src/main/resources/es7/customer.yml | 46 + .../src/main/resources/es7/mytest_user.yml | 15 + .../client/adapter/es7x/test/ES7xTest.java | 119 +++ .../adapter/es7x/test/ESConnectionTest.java | 45 + .../adapter/es7x/test/TestConstant.java | 40 + client-adapter/launcher/pom.xml | 15 +- .../launcher/src/main/assembly/dev.xml | 11 +- .../launcher/src/main/assembly/release.xml | 13 +- client-adapter/pom.xml | 3 + 54 files changed, 7172 insertions(+), 6 deletions(-) create mode 100644 client-adapter/es-core/pom.xml create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfig.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SchemaItem.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESBulkRequest.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java create mode 100644 client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESTemplate.java create mode 100644 client-adapter/es6x/pom.xml create mode 100644 client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java create mode 100644 client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java create mode 100644 client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java create mode 100644 client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java create mode 100644 client-adapter/es6x/src/main/java/org/elasticsearch/client/RequestConvertersExt.java create mode 100644 client-adapter/es6x/src/main/java/org/elasticsearch/client/RestHighLevelClientExt.java create mode 100644 client-adapter/es6x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter create mode 100644 client-adapter/es6x/src/main/resources/es6/biz_order.yml create mode 100644 client-adapter/es6x/src/main/resources/es6/customer.yml create mode 100644 client-adapter/es6x/src/main/resources/es6/mytest_user.yml create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ConfigLoadTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ESTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/SqlParseTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/TestConstant.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/Common.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/db_schema.sql create mode 100644 client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/es_mapping.json create mode 100644 client-adapter/es6x/src/test/resources/es6/mytest_user_single.yml_ create mode 100644 client-adapter/es6x/src/test/resources/log4j2-test.xml create mode 100644 client-adapter/es6x/src/test/resources/logback-test.xml create mode 100644 client-adapter/es7x/pom.xml create mode 100644 client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java create mode 100644 client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/etl/ESEtlService.java create mode 100644 client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java create mode 100644 client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java create mode 100644 client-adapter/es7x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter create mode 100644 client-adapter/es7x/src/main/resources/es7/biz_order.yml create mode 100644 client-adapter/es7x/src/main/resources/es7/customer.yml create mode 100644 client-adapter/es7x/src/main/resources/es7/mytest_user.yml create mode 100644 client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java create mode 100644 client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java create mode 100644 client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/TestConstant.java diff --git a/client-adapter/es-core/pom.xml b/client-adapter/es-core/pom.xml new file mode 100644 index 00000000..af8aaacc --- /dev/null +++ b/client-adapter/es-core/pom.xml @@ -0,0 +1,49 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.5-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.es-core + jar + canal client adapter es-core module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.common + ${project.version} + provided + + + com.alibaba.fastsql + fastsql + 2.0.0_preview_855 + + + + junit + junit + 4.12 + test + + + mysql + mysql-connector-java + 5.1.40 + test + + + com.h2database + h2 + 1.4.192 + test + + + + diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java new file mode 100644 index 00000000..d5bf39dc --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java @@ -0,0 +1,177 @@ +package com.alibaba.otter.canal.client.adapter.es.core; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; +import org.apache.commons.lang.StringUtils; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser; +import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor; +import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +/** + * ES外部适配器 + * + * @author rewerma 2018-10-20 + * @version 1.0.0 + */ +public abstract class ESAdapter implements OuterAdapter { + + protected Map esSyncConfig = new ConcurrentHashMap<>(); // 文件名对应配置 + protected Map> dbTableEsSyncConfig = new ConcurrentHashMap<>(); // schema-table对应配置 + + protected ESTemplate esTemplate; + + protected ESSyncService esSyncService; + + protected ESConfigMonitor esConfigMonitor; + + protected Properties envProperties; + + public ESSyncService getEsSyncService() { + return esSyncService; + } + + public Map getEsSyncConfig() { + return esSyncConfig; + } + + public Map> getDbTableEsSyncConfig() { + return dbTableEsSyncConfig; + } + + @Override + public void init(OuterAdapterConfig configuration, Properties envProperties) { + try { + this.envProperties = envProperties; + Map esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); + // 过滤不匹配的key的配置 + esSyncConfigTmp.forEach((key, config) -> { + if ((config.getOuterAdapterKey() == null && configuration.getKey() == null) + || (config.getOuterAdapterKey() != null + && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { + esSyncConfig.put(key, config); + } + }); + + for (Map.Entry entry : esSyncConfig.entrySet()) { + String configName = entry.getKey(); + ESSyncConfig config = entry.getValue(); + + addSyncConfigToCache(configName, config); + } + + esSyncService = new ESSyncService(esTemplate); + + esConfigMonitor = new ESConfigMonitor(); + esConfigMonitor.init(this, envProperties); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void sync(List dmls) { + if (dmls == null || dmls.isEmpty()) { + return; + } + for (Dml dml : dmls) { + if (!dml.getIsDdl()) { + sync(dml); + } + } + esSyncService.commit(); // 批次统一提交 + + } + + private void sync(Dml dml) { + String database = dml.getDatabase(); + String table = dml.getTable(); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + + "_" + database + "-" + table); + } else { + configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table); + } + + if (configMap != null && !configMap.values().isEmpty()) { + esSyncService.sync(configMap.values(), dml); + } + } + + @Override + public abstract EtlResult etl(String task, List params); + + @Override + public abstract Map count(String task); + + @Override + public void destroy() { + if (esConfigMonitor != null) { + esConfigMonitor.destroy(); + } + } + + @Override + public String getDestination(String task) { + ESSyncConfig config = esSyncConfig.get(task); + if (config != null) { + return config.getDestination(); + } + return null; + } + + public void addSyncConfigToCache(String configName, ESSyncConfig config) { + Properties envProperties = this.envProperties; + SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql()); + config.getEsMapping().setSchemaItem(schemaItem); + + DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource == null || dataSource.getUrl() == null) { + throw new RuntimeException("No data source found: " + config.getDataSourceKey()); + } + Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); + Matcher matcher = pattern.matcher(dataSource.getUrl()); + if (!matcher.find()) { + throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); + } + String schema = matcher.group(2); + + schemaItem.getAliasTableItems().values().forEach(tableItem -> { + Map esSyncConfigMap; + if (envProperties != null + && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + esSyncConfigMap = dbTableEsSyncConfig + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-" + + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-" + + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); + } else { + esSyncConfigMap = dbTableEsSyncConfig + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-" + + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); + } + + esSyncConfigMap.put(configName, config); + }); + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfig.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfig.java new file mode 100644 index 00000000..1277ebbc --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfig.java @@ -0,0 +1,251 @@ +package com.alibaba.otter.canal.client.adapter.es.core.config; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.support.AdapterConfig; + +/** + * ES 映射配置 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncConfig implements AdapterConfig { + + private String dataSourceKey; // 数据源key + + private String outerAdapterKey; // adapter key + + private String groupId; // group id + + private String destination; // canal destination + + private ESMapping esMapping; + + private String esVersion = "es6"; + + public void validate() { + if (esMapping._index == null) { + throw new NullPointerException("esMapping._index"); + } + if ("es6".equals(esVersion) && esMapping._type == null) { + throw new NullPointerException("esMapping._type"); + } + if (esMapping._id == null && esMapping.getPk() == null) { + throw new NullPointerException("esMapping._id or esMapping.pk"); + } + if (esMapping.sql == null) { + throw new NullPointerException("esMapping.sql"); + } + } + + public String getDataSourceKey() { + return dataSourceKey; + } + + public void setDataSourceKey(String dataSourceKey) { + this.dataSourceKey = dataSourceKey; + } + + public String getOuterAdapterKey() { + return outerAdapterKey; + } + + public void setOuterAdapterKey(String outerAdapterKey) { + this.outerAdapterKey = outerAdapterKey; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public ESMapping getEsMapping() { + return esMapping; + } + + public void setEsMapping(ESMapping esMapping) { + this.esMapping = esMapping; + } + + public ESMapping getMapping() { + return esMapping; + } + + public String getEsVersion() { + return esVersion; + } + + public void setEsVersion(String esVersion) { + this.esVersion = esVersion; + } + + public static class ESMapping implements AdapterMapping { + + private String _index; + private String _type; + private String _id; + private boolean upsert = false; + private String pk; + private Map relations = new LinkedHashMap<>(); + private String sql; + // 对象字段, 例: objFields: + // - _labels: array:; + private Map objFields = new LinkedHashMap<>(); + private List skips = new ArrayList<>(); + private int commitBatch = 1000; + private String etlCondition; + private boolean syncByTimestamp = false; // 是否按时间戳定时同步 + private Long syncInterval; // 同步时间间隔 + + private SchemaItem schemaItem; // sql解析结果模型 + + public String get_index() { + return _index; + } + + public void set_index(String _index) { + this._index = _index; + } + + public String get_type() { + return _type; + } + + public void set_type(String _type) { + this._type = _type; + } + + public String get_id() { + return _id; + } + + public void set_id(String _id) { + this._id = _id; + } + + public boolean isUpsert() { + return upsert; + } + + public void setUpsert(boolean upsert) { + this.upsert = upsert; + } + + public String getPk() { + return pk; + } + + public void setPk(String pk) { + this.pk = pk; + } + + public Map getObjFields() { + return objFields; + } + + public void setObjFields(Map objFields) { + this.objFields = objFields; + } + + public List getSkips() { + return skips; + } + + public void setSkips(List skips) { + this.skips = skips; + } + + public Map getRelations() { + return relations; + } + + public void setRelations(Map relations) { + this.relations = relations; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public int getCommitBatch() { + return commitBatch; + } + + public void setCommitBatch(int commitBatch) { + this.commitBatch = commitBatch; + } + + public String getEtlCondition() { + return etlCondition; + } + + public void setEtlCondition(String etlCondition) { + this.etlCondition = etlCondition; + } + + public Long getSyncInterval() { + return syncInterval; + } + + public void setSyncInterval(Long syncInterval) { + this.syncInterval = syncInterval; + } + + public boolean isSyncByTimestamp() { + return syncByTimestamp; + } + + public void setSyncByTimestamp(boolean syncByTimestamp) { + this.syncByTimestamp = syncByTimestamp; + } + + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } + } + + public static class RelationMapping { + + private String name; + private String parent; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java new file mode 100644 index 00000000..6b717a5d --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java @@ -0,0 +1,47 @@ +package com.alibaba.otter.canal.client.adapter.es.core.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; + +/** + * ES 配置装载器 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); + + public static synchronized Map load(Properties envProperties) { + logger.info("## Start loading es mapping config ... "); + + Map esSyncConfig = new LinkedHashMap<>(); + + String esv = envProperties.getProperty("es.version"); + Map configContentMap = MappingConfigsLoader.loadConfigs(esv); + configContentMap.forEach((fileName, content) -> { + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties); + if (config == null) { + return; + } + config.setEsVersion(esv); + try { + config.validate(); + } catch (Exception e) { + throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); + } + esSyncConfig.put(fileName, config); + }); + + logger.info("## ES mapping config loaded"); + return esSyncConfig; + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SchemaItem.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SchemaItem.java new file mode 100644 index 00000000..441e440f --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SchemaItem.java @@ -0,0 +1,439 @@ +package com.alibaba.otter.canal.client.adapter.es.core.config; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + + +/** + * ES 映射配置视图 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class SchemaItem { + + private Map aliasTableItems = new LinkedHashMap<>(); // 别名对应表名 + private Map selectFields = new LinkedHashMap<>(); // 查询字段 + private String sql; + + private volatile Map> tableItemAliases; + private volatile Map> columnFields; + private volatile Boolean allFieldsSimple; + + public void init() { + this.getTableItemAliases(); + this.getColumnFields(); + this.isAllFieldsSimple(); + aliasTableItems.values().forEach(tableItem -> { + tableItem.getRelationTableFields(); + tableItem.getRelationSelectFieldItems(); + }); + } + + public Map getAliasTableItems() { + return aliasTableItems; + } + + public void setAliasTableItems(Map aliasTableItems) { + this.aliasTableItems = aliasTableItems; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Map getSelectFields() { + return selectFields; + } + + public void setSelectFields(Map selectFields) { + this.selectFields = selectFields; + } + + public String toSql() { + // todo + return null; + } + + public Map> getTableItemAliases() { + if (tableItemAliases == null) { + synchronized (SchemaItem.class) { + if (tableItemAliases == null) { + tableItemAliases = new LinkedHashMap<>(); + aliasTableItems.forEach((alias, tableItem) -> { + List aliases = tableItemAliases + .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>()); + aliases.add(tableItem); + }); + } + } + } + return tableItemAliases; + } + + public Map> getColumnFields() { + if (columnFields == null) { + synchronized (SchemaItem.class) { + if (columnFields == null) { + columnFields = new LinkedHashMap<>(); + getSelectFields() + .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { + // TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); + // if (!tableItem.isSubQuery()) { + //当数据列并非原始列时，columnName是空的，例如concat('px',id) + if(columnItem.getColumnName() != null) { + List fieldItems = columnFields.computeIfAbsent( + columnItem.getOwner() + "." + columnItem.getColumnName(), + k -> new ArrayList<>()); + fieldItems.add(fieldItem); + } + // } else { + // tableItem.getSubQueryFields().forEach(subQueryField -> { + // List fieldItems = columnFields.computeIfAbsent( + // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), + // k -> new ArrayList<>()); + // fieldItems.add(fieldItem); + // }); + // } + })); + } + } + } + return columnFields; + } + + public boolean isAllFieldsSimple() { + if (allFieldsSimple == null) { + synchronized (SchemaItem.class) { + if (allFieldsSimple == null) { + allFieldsSimple = true; + + for (FieldItem fieldItem : getSelectFields().values()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + } + } + } + + return allFieldsSimple; + } + + public TableItem getMainTable() { + if (!aliasTableItems.isEmpty()) { + return aliasTableItems.values().iterator().next(); + } else { + return null; + } + } + + public FieldItem getIdFieldItem(ESSyncConfig.ESMapping mapping) { + if (mapping.get_id() != null) { + return getSelectFields().get(mapping.get_id()); + } else { + return getSelectFields().get(mapping.getPk()); + } + } + + public static class TableItem { + + private SchemaItem schemaItem; + + private String schema; + private String tableName; + private String alias; + private String subQuerySql; + private List subQueryFields = new ArrayList<>(); + private List relationFields = new ArrayList<>(); + + private boolean main; + private boolean subQuery; + + private volatile Map> relationTableFields; // 当前表关联条件字段对应主表查询字段 + private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 + + public TableItem(SchemaItem schemaItem){ + this.schemaItem = schemaItem; + } + + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getSubQuerySql() { + return subQuerySql; + } + + public void setSubQuerySql(String subQuerySql) { + this.subQuerySql = subQuerySql; + } + + public boolean isMain() { + return main; + } + + public void setMain(boolean main) { + this.main = main; + } + + public boolean isSubQuery() { + return subQuery; + } + + public void setSubQuery(boolean subQuery) { + this.subQuery = subQuery; + } + + public List getSubQueryFields() { + return subQueryFields; + } + + public void setSubQueryFields(List subQueryFields) { + this.subQueryFields = subQueryFields; + } + + public List getRelationFields() { + return relationFields; + } + + public void setRelationFields(List relationFields) { + this.relationFields = relationFields; + } + + public Map> getRelationTableFields() { + if (relationTableFields == null) { + synchronized (SchemaItem.class) { + if (relationTableFields == null) { + relationTableFields = new LinkedHashMap<>(); + + getRelationFields().forEach(relationFieldsPair -> { + FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); + FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + FieldItem currentTableRelField = null; + if (getAlias().equals(leftFieldItem.getOwner())) { + currentTableRelField = leftFieldItem; + } else if (getAlias().equals(rightFieldItem.getOwner())) { + currentTableRelField = rightFieldItem; + } + + if (currentTableRelField != null) { + List selectFieldItem = getSchemaItem().getColumnFields() + .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + selectFieldItem = getSchemaItem().getColumnFields() + .get(rightFieldItem.getOwner() + "." + + rightFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + throw new UnsupportedOperationException( + "Relation condition column must in select columns."); + } + } + } + }); + } + } + } + return relationTableFields; + } + + public List getRelationSelectFieldItems() { + if (relationSelectFieldItems == null) { + synchronized (SchemaItem.class) { + if (relationSelectFieldItems == null) { + List relationSelectFieldItemsTmp = new ArrayList<>(); + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getOwners().contains(getAlias())) { + relationSelectFieldItemsTmp.add(fieldItem); + } + } + relationSelectFieldItems = relationSelectFieldItemsTmp; + } + } + } + return relationSelectFieldItems; + } + } + + public static class RelationFieldsPair { + + private FieldItem leftFieldItem; + private FieldItem rightFieldItem; + + public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){ + this.leftFieldItem = leftFieldItem; + this.rightFieldItem = rightFieldItem; + } + + public FieldItem getLeftFieldItem() { + return leftFieldItem; + } + + public void setLeftFieldItem(FieldItem leftFieldItem) { + this.leftFieldItem = leftFieldItem; + } + + public FieldItem getRightFieldItem() { + return rightFieldItem; + } + + public void setRightFieldItem(FieldItem rightFieldItem) { + this.rightFieldItem = rightFieldItem; + } + } + + public static class FieldItem { + + private String fieldName; + private String expr; + private List columnItems = new ArrayList<>(); + private List owners = new ArrayList<>(); + + private boolean method; + private boolean binaryOp; + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public String getExpr() { + return expr; + } + + public void setExpr(String expr) { + this.expr = expr; + } + + public List getColumnItems() { + return columnItems; + } + + public void setColumnItems(List columnItems) { + this.columnItems = columnItems; + } + + public boolean isMethod() { + return method; + } + + public void setMethod(boolean method) { + this.method = method; + } + + public boolean isBinaryOp() { + return binaryOp; + } + + public void setBinaryOp(boolean binaryOp) { + this.binaryOp = binaryOp; + } + + public List getOwners() { + return owners; + } + + public void setOwners(List owners) { + this.owners = owners; + } + + public void addColumn(ColumnItem columnItem) { + columnItems.add(columnItem); + } + + public ColumnItem getColumn() { + if (!columnItems.isEmpty()) { + return columnItems.get(0); + } else { + return null; + } + } + + public String getOwner() { + if (!owners.isEmpty()) { + return owners.get(0); + } else { + return null; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldItem fieldItem = (FieldItem) o; + + return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null; + } + + @Override + public int hashCode() { + return fieldName != null ? fieldName.hashCode() : 0; + } + } + + public static class ColumnItem { + + private String owner; + private String columnName; + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java new file mode 100644 index 00000000..59d94512 --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java @@ -0,0 +1,276 @@ +package com.alibaba.otter.canal.client.adapter.es.core.config; + +import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd; +import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import com.alibaba.fastsql.sql.SQLUtils; +import com.alibaba.fastsql.sql.ast.SQLExpr; +import com.alibaba.fastsql.sql.ast.expr.*; +import com.alibaba.fastsql.sql.ast.statement.*; +import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; +import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.fastsql.sql.parser.ParserException; +import com.alibaba.fastsql.sql.parser.SQLStatementParser; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.RelationFieldsPair; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem; + +/** + * ES同步指定sql格式解析 + * + * @author rewerma 2018-10-26 下午03:45:49 + * @version 1.0.0 + */ +public class SqlParser { + + /** + * 解析sql + * + * @param sql sql + * @return 视图对象 + */ + public static SchemaItem parse(String sql) { + try { + SQLStatementParser parser = new MySqlStatementParser(sql); + SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement(); + MySqlSelectQueryBlock sqlSelectQueryBlock = (MySqlSelectQueryBlock) statement.getSelect().getQuery(); + + SchemaItem schemaItem = new SchemaItem(); + schemaItem.setSql(SQLUtils.toMySqlString(sqlSelectQueryBlock)); + SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom(); + List tableItems = new ArrayList<>(); + SqlParser.visitSelectTable(schemaItem, sqlTableSource, tableItems, null); + tableItems.forEach(tableItem -> schemaItem.getAliasTableItems().put(tableItem.getAlias(), tableItem)); + + List fieldItems = collectSelectQueryFields(sqlSelectQueryBlock); + fieldItems.forEach(fieldItem -> schemaItem.getSelectFields().put(fieldItem.getFieldName(), fieldItem)); + + schemaItem.init(); + + if (schemaItem.getAliasTableItems().isEmpty() || schemaItem.getSelectFields().isEmpty()) { + throw new ParserException("Parse sql error"); + } + return schemaItem; + } catch (Exception e) { + throw new ParserException(); + } + } + + /** + * 归集字段 + * + * @param sqlSelectQueryBlock sqlSelectQueryBlock + * @return 字段属性列表 + */ + private static List collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) { + return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> { + FieldItem fieldItem = new FieldItem(); + fieldItem.setFieldName(selectItem.getAlias()); + fieldItem.setExpr(selectItem.toString()); + visitColumn(selectItem.getExpr(), fieldItem); + return fieldItem; + }).collect(Collectors.toList()); + } + + /** + * 解析字段 + * + * @param expr sql expr + * @param fieldItem 字段属性 + */ + private static void visitColumn(SQLExpr expr, FieldItem fieldItem) { + if (expr instanceof SQLIdentifierExpr) { + // 无owner + SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr; + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(identifierExpr.getName()); + fieldItem.setExpr(identifierExpr.toString()); + } + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(identifierExpr.getName()); + fieldItem.getOwners().add(null); + fieldItem.addColumn(columnItem); + } else if (expr instanceof SQLPropertyExpr) { + // 有owner + SQLPropertyExpr sqlPropertyExpr = (SQLPropertyExpr) expr; + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(sqlPropertyExpr.getName()); + fieldItem.setExpr(sqlPropertyExpr.toString()); + } + fieldItem.getOwners().add(sqlPropertyExpr.getOwnernName()); + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(sqlPropertyExpr.getName()); + columnItem.setOwner(sqlPropertyExpr.getOwnernName()); + fieldItem.addColumn(columnItem); + } else if (expr instanceof SQLMethodInvokeExpr) { + SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr) expr; + fieldItem.setMethod(true); + for (SQLExpr sqlExpr : methodInvokeExpr.getArguments()) { + visitColumn(sqlExpr, fieldItem); + } + } else if (expr instanceof SQLBinaryOpExpr) { + SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr; + fieldItem.setBinaryOp(true); + visitColumn(sqlBinaryOpExpr.getLeft(), fieldItem); + visitColumn(sqlBinaryOpExpr.getRight(), fieldItem); + } else if (expr instanceof SQLCaseExpr) { + SQLCaseExpr sqlCaseExpr = (SQLCaseExpr) expr; + fieldItem.setMethod(true); + sqlCaseExpr.getItems().forEach(item -> visitColumn(item.getConditionExpr(), fieldItem)); + } else if (expr instanceof SQLCharExpr) { + SQLCharExpr sqlCharExpr = (SQLCharExpr) expr; + String owner = null; + String columnName = null; + if (sqlCharExpr.getParent() instanceof SQLCaseExpr.Item) { + owner = ((SQLPropertyExpr) ((SQLCaseExpr.Item) sqlCharExpr.getParent()).getValueExpr()).getOwnernName(); + columnName = ((SQLPropertyExpr) ((SQLCaseExpr.Item) sqlCharExpr.getParent()).getValueExpr()).getName(); + } + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(columnName); + fieldItem.setExpr(sqlCharExpr.toString()); + } + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(columnName); + columnItem.setOwner(owner); + fieldItem.getOwners().add(owner); + fieldItem.addColumn(columnItem); + } + } + + /** + * 解析表 + * + * @param schemaItem 视图对象 + * @param sqlTableSource sqlTableSource + * @param tableItems 表对象列表 + * @param tableItemTmp 表对象(临时) + */ + private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource, + List tableItems, TableItem tableItemTmp) { + if (sqlTableSource instanceof SQLExprTableSource) { + SQLExprTableSource sqlExprTableSource = (SQLExprTableSource) sqlTableSource; + TableItem tableItem; + if (tableItemTmp != null) { + tableItem = tableItemTmp; + } else { + tableItem = new TableItem(schemaItem); + } + tableItem.setSchema(sqlExprTableSource.getSchema()); + tableItem.setTableName(sqlExprTableSource.getTableName()); + if (tableItem.getAlias() == null) { + tableItem.setAlias(sqlExprTableSource.getAlias()); + } + if (tableItems.isEmpty()) { + // 第一张表为主表 + tableItem.setMain(true); + } + tableItems.add(tableItem); + } else if (sqlTableSource instanceof SQLJoinTableSource) { + SQLJoinTableSource sqlJoinTableSource = (SQLJoinTableSource) sqlTableSource; + SQLTableSource leftTableSource = sqlJoinTableSource.getLeft(); + visitSelectTable(schemaItem, leftTableSource, tableItems, null); + SQLTableSource rightTableSource = sqlJoinTableSource.getRight(); + TableItem rightTableItem = new TableItem(schemaItem); + // 解析on条件字段 + visitOnCondition(sqlJoinTableSource.getCondition(), rightTableItem); + visitSelectTable(schemaItem, rightTableSource, tableItems, rightTableItem); + + } else if (sqlTableSource instanceof SQLSubqueryTableSource) { + SQLSubqueryTableSource subQueryTableSource = (SQLSubqueryTableSource) sqlTableSource; + MySqlSelectQueryBlock sqlSelectQuery = (MySqlSelectQueryBlock) subQueryTableSource.getSelect().getQuery(); + TableItem tableItem; + if (tableItemTmp != null) { + tableItem = tableItemTmp; + } else { + tableItem = new TableItem(schemaItem); + } + tableItem.setAlias(subQueryTableSource.getAlias()); + tableItem.setSubQuerySql(SQLUtils.toMySqlString(sqlSelectQuery)); + tableItem.setSubQuery(true); + tableItem.setSubQueryFields(collectSelectQueryFields(sqlSelectQuery)); + visitSelectTable(schemaItem, sqlSelectQuery.getFrom(), tableItems, tableItem); + } + } + + /** + * 解析on条件 + * + * @param expr sql expr + * @param tableItem 表对象 + */ + private static void visitOnCondition(SQLExpr expr, TableItem tableItem) { + if (!(expr instanceof SQLBinaryOpExpr)) { + throw new UnsupportedOperationException(); + } + SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr; + if (sqlBinaryOpExpr.getOperator() == BooleanAnd) { + visitOnCondition(sqlBinaryOpExpr.getLeft(), tableItem); + visitOnCondition(sqlBinaryOpExpr.getRight(), tableItem); + } else if (sqlBinaryOpExpr.getOperator() == Equality) { + FieldItem leftFieldItem = new FieldItem(); + visitColumn(sqlBinaryOpExpr.getLeft(), leftFieldItem); + if (leftFieldItem.getColumnItems().size() != 1 || leftFieldItem.isMethod() || leftFieldItem.isBinaryOp()) { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + FieldItem rightFieldItem = new FieldItem(); + visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem); + if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() + || rightFieldItem.isBinaryOp()) { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem)); + } else { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + } + + public static MySqlSelectQueryBlock parseSQLSelectQueryBlock(String sql) { + if (sql == null || "".equals(sql)) { + return null; + } + SQLStatementParser parser = new MySqlStatementParser(sql); + SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement(); + return (MySqlSelectQueryBlock) statement.getSelect().getQuery(); + } + + public static String parse4SQLSelectItem(MySqlSelectQueryBlock sqlSelectQueryBlock) { + List selectItems = sqlSelectQueryBlock.getSelectList(); + StringBuilder subSql = new StringBuilder(); + int i = 0; + for (SQLSelectItem sqlSelectItem : selectItems) { + if (i != 0) { + subSql.append(","); + } else { + i++; + } + subSql.append(SQLUtils.toMySqlString(sqlSelectItem)); + } + return subSql.toString(); + } + + public static String parse4FromTableSource(MySqlSelectQueryBlock sqlSelectQueryBlock) { + SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom(); + return SQLUtils.toMySqlString(sqlTableSource); + } + + public static String parse4WhereItem(MySqlSelectQueryBlock sqlSelectQueryBlock) { + SQLExpr sqlExpr = sqlSelectQueryBlock.getWhere(); + if (sqlExpr != null) { + return SQLUtils.toMySqlString(sqlExpr); + } + return null; + } + + public static String parse4GroupBy(MySqlSelectQueryBlock sqlSelectQueryBlock) { + SQLSelectGroupByClause expr = sqlSelectQueryBlock.getGroupBy(); + if (expr != null) { + return SQLUtils.toMySqlString(expr); + } + return null; + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java new file mode 100644 index 00000000..527d292b --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java @@ -0,0 +1,146 @@ +package com.alibaba.otter.canal.client.adapter.es.core.monitor; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class ESConfigMonitor { + + private static final Logger logger = LoggerFactory.getLogger(ESConfigMonitor.class); + + private static final String adapterName = "es"; + + private ESAdapter esAdapter; + + private Properties envProperties; + + private FileAlterationMonitor fileMonitor; + + public void init(ESAdapter esAdapter, Properties envProperties) { + this.esAdapter = esAdapter; + this.envProperties = envProperties; + File confDir = Util.getConfDirPath(adapterName); + try { + FileAlterationObserver observer = new FileAlterationObserver(confDir, + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileListener listener = new FileListener(); + observer.addListener(listener); + fileMonitor = new FileAlterationMonitor(3000, observer); + fileMonitor.start(); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + public void destroy() { + try { + fileMonitor.stop(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private class FileListener extends FileAlterationListenerAdaptor { + + @Override + public void onFileCreate(File file) { + super.onFileCreate(file); + try { + // 加载新增的配置文件 + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); + ESSyncConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties); + if (config != null) { + config.validate(); + addConfigToCache(file, config); + logger.info("Add a new es mapping config: {} to canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileChange(File file) { + super.onFileChange(file); + + try { + if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { + // 加载配置文件 + String configContent = MappingConfigsLoader + .loadConfig(adapterName + File.separator + file.getName()); + if (configContent == null) { + onFileDelete(file); + return; + } + ESSyncConfig config = YmlConfigBinder + .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties); + if (config == null) { + return; + } + config.validate(); + if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { + deleteConfigFromCache(file); + } + addConfigToCache(file, config); + + logger.info("Change a es mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileDelete(File file) { + super.onFileDelete(file); + + try { + if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { + deleteConfigFromCache(file); + + logger.info("Delete a es mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void addConfigToCache(File file, ESSyncConfig config) { + esAdapter.getEsSyncConfig().put(file.getName(), config); + + esAdapter.addSyncConfigToCache(file.getName(),config); + } + + private void deleteConfigFromCache(File file) { + esAdapter.getEsSyncConfig().remove(file.getName()); + for (Map configMap : esAdapter.getDbTableEsSyncConfig().values()) { + if (configMap != null) { + configMap.remove(file.getName()); + } + } + + } + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java new file mode 100644 index 00000000..349093bb --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java @@ -0,0 +1,875 @@ +package com.alibaba.otter.canal.client.adapter.es.core.service; + +import java.util.*; + +import javax.sql.DataSource; + +import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; +import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.Util; + +/** + * ES 同步 Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncService { + + private static Logger logger = LoggerFactory.getLogger(ESSyncService.class); + + private ESTemplate esTemplate; + + public ESSyncService(ESTemplate esTemplate){ + this.esTemplate = esTemplate; + } + + public void sync(Collection esSyncConfigs, Dml dml) { + long begin = System.currentTimeMillis(); + if (esSyncConfigs != null) { + if (logger.isTraceEnabled()) { + logger.trace("Destination: {}, database:{}, table:{}, type:{}, affected index count: {}", + dml.getDestination(), + dml.getDatabase(), + dml.getTable(), + dml.getType(), + esSyncConfigs.size()); + } + + for (ESSyncConfig config : esSyncConfigs) { + if (logger.isTraceEnabled()) { + logger.trace("Prepared to sync index: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } + this.sync(config, dml); + if (logger.isTraceEnabled()) { + logger.trace("Sync completed: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } + } + if (logger.isTraceEnabled()) { + logger.trace("Sync elapsed time: {} ms, affected indexes count：{}, destination: {}", + (System.currentTimeMillis() - begin), + esSyncConfigs.size(), + dml.getDestination()); + } + if (logger.isDebugEnabled()) { + StringBuilder configIndexes = new StringBuilder(); + esSyncConfigs + .forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().get_index()).append(" ")); + logger.debug("DML: {} \nAffected indexes: {}", + JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue), + configIndexes.toString()); + } + } + } + + public void sync(ESSyncConfig config, Dml dml) { + try { + // 如果是按时间戳定时更新则返回 + if (config.getEsMapping().isSyncByTimestamp()) { + return; + } + + long begin = System.currentTimeMillis(); + + String type = dml.getType(); + if (type != null && type.equalsIgnoreCase("INSERT")) { + insert(config, dml); + } else if (type != null && type.equalsIgnoreCase("UPDATE")) { + update(config, dml); + } else if (type != null && type.equalsIgnoreCase("DELETE")) { + delete(config, dml); + } else { + return; + } + + if (logger.isTraceEnabled()) { + logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}", + (System.currentTimeMillis() - begin), + dml.getDestination(), + config.getEsMapping().get_index()); + } + } catch (Throwable e) { + logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml); + throw new RuntimeException(e); + } + } + + /** + * 插入操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void insert(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledInsert(config, dml, data); + } else { + // ------是主表 查询sql来插入------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + mainTableInsert(config, dml, data); + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段插入------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); + } + + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段插入------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段插入 执行全sql更新es------ + wholeSqlOperation(config, dml, data, null, tableItem); + } + } + } + } + } + + /** + * 更新操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void update(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + List> oldList = dml.getOld(); + if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + int i = 0; + for (Map data : dataList) { + Map old = oldList.get(i); + if (data == null || data.isEmpty() || old == null || old.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + // ------主表 查询sql来更新------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + ESMapping mapping = config.getEsMapping(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName); + + boolean idFieldSimple = true; + if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) { + idFieldSimple = false; + } + + boolean allUpdateFieldSimple = true; + out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allUpdateFieldSimple = false; + break out; + } + } + } + } + + // 不支持主键更新!! + + // 判断是否有外键更新 + boolean fkChanged = false; + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + boolean changed = false; + for (List fieldItems : tableItem.getRelationTableFields().values()) { + for (FieldItem fieldItem : fieldItems) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + fkChanged = true; + changed = true; + break; + } + } + } + // 如果外键有修改,则更新所对应该表的所有查询条件数据 + if (changed) { + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + fieldItem.getColumnItems() + .forEach(columnItem -> old.put(columnItem.getColumnName(), null)); + } + } + } + + // 判断主键和所更新的字段是否全为简单字段 + if (idFieldSimple && allUpdateFieldSimple && !fkChanged) { + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + mainTableUpdate(config, dml, data, old); + } + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); + } + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, old, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + wholeSqlOperation(config, dml, data, old, tableItem); + } + } + } + + i++; + } + } + + /** + * 删除操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void delete(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + ESMapping mapping = config.getEsMapping(); + + // ------是主表------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + if (mapping.get_id() != null) { + FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping); + // 主键为简单字段 + if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) { + Object idVal = esTemplate.getValFromData(mapping, + data, + idFieldItem.getFieldName(), + idFieldItem.getColumn().getColumnName()); + + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + esTemplate.delete(mapping, idVal, null); + } else { + // ------主键带函数, 查询sql获取主键删除------ + // FIXME 删除时反查sql为空记录, 无法获获取 id field 值 + mainTableDelete(config, dml, data); + } + } else { + FieldItem pkFieldItem = schemaItem.getIdFieldItem(mapping); + if (!pkFieldItem.isMethod() && !pkFieldItem.isBinaryOp()) { + Map esFieldData = new LinkedHashMap<>(); + Object pkVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + pkVal); + } + esFieldData.remove(pkFieldItem.getFieldName()); + esFieldData.keySet().forEach(key -> esFieldData.put(key, null)); + esTemplate.delete(mapping, pkVal, esFieldData); + } else { + // ------主键带函数, 查询sql获取主键删除------ + mainTableDelete(config, dml, data); + } + } + + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新为null------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), null); + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + wholeSqlOperation(config, dml, data, null, tableItem); + } + } + } + } + + /** + * 单表简单字段insert + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Single table insert to es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + esTemplate.insert(mapping, idVal, esFieldData); + } + + /** + * 主表(单表)复杂字段insert + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void mainTableInsert(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table insert to es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + Util.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table insert to es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + esTemplate.insert(mapping, idVal, esFieldData); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + private void mainTableDelete(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + Util.sqlRS(ds, sql, rs -> { + try { + Map esFieldData = null; + if (mapping.getPk() != null) { + esFieldData = new LinkedHashMap<>(); + esTemplate.getESDataFromDmlData(mapping, data, esFieldData); + esFieldData.remove(mapping.getPk()); + for (String key : esFieldData.keySet()) { + esFieldData.put(Util.cleanColumn(key), null); + } + } + while (rs.next()) { + Object idVal = esTemplate.getIdValFromRS(mapping, rs); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table delete to es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + esTemplate.delete(mapping, idVal, esFieldData); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 关联表主表简单字段operation + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, + TableItem tableItem, Map esFieldData) { + ESMapping mapping = config.getEsMapping(); + + Map paramsTmp = new LinkedHashMap<>(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromData(mapping, + data, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + paramsTmp.put(fieldName, value); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + esTemplate.updateByQuery(config, paramsTmp, esFieldData); + } + + /** + * 关联子查询, 主表简单字段operation + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param old 单行old数据 + * @param tableItem 当前表配置 + */ + private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, + Map old, TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + + MySqlSelectQueryBlock queryBlock = SqlParser.parseSQLSelectQueryBlock(tableItem.getSubQuerySql()); + StringBuilder sql = new StringBuilder(); + sql.append("SELECT ") + .append(SqlParser.parse4SQLSelectItem(queryBlock)) + .append(" FROM ") + .append(SqlParser.parse4FromTableSource(queryBlock)); + + String whereSql = SqlParser.parse4WhereItem(queryBlock); + if (whereSql != null) { + sql.append(" WHERE ").append(whereSql); + } else { + sql.append(" WHERE 1=1 "); + } + + List values = new ArrayList<>(); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + sql.append(" AND ").append(columnName).append("=? "); + values.add(value); + } + + String groupSql = SqlParser.parse4GroupBy(queryBlock); + if (groupSql != null) { + sql.append(groupSql); + } + + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + Util.sqlRS(ds, sql.toString(), values, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old != null) { + out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) { + for (ColumnItem columnItem0 : fieldItem.getColumnItems()) { + if (fieldItem1.getFieldName().equals(columnItem0.getColumnName())) + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val); + break out; + } + } + } + } + } else { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val); + } + } + + Map paramsTmp = new LinkedHashMap<>(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + paramsTmp.put(fieldName, value); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + esTemplate.updateByQuery(config, paramsTmp, esFieldData); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 关联(子查询), 主表复杂字段operation, 全sql执行 + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map data, Map old, + TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + // 防止最后出现groupby 导致sql解析异常 + String[] sqlSplit = mapping.getSql().split("GROUP\\ BY(?!(.*)ON)"); + String sqlNoWhere = sqlSplit[0]; + + String sqlGroupBy = ""; + + if (sqlSplit.length > 1) { + sqlGroupBy = "GROUP BY " + sqlSplit[1]; + } + + StringBuilder sql = new StringBuilder(sqlNoWhere + " WHERE "); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName); + } + int len = sql.length(); + sql.delete(len - 5, len); + sql.append(sqlGroupBy); + + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + Util.sqlRS(ds, sql.toString(), rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old != null) { + // 从表子查询 + out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) { + for (ColumnItem columnItem0 : fieldItem.getColumnItems()) { + if (fieldItem1.getFieldName().equals(columnItem0.getColumnName())) + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + break out; + } + } + } + } + // 从表非子查询 + for (FieldItem fieldItem1 : tableItem.getRelationSelectFieldItems()) { + if (fieldItem1.equals(fieldItem)) { + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getFieldName()); + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val); + break; + } + } + } + } + } else { + Object val = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val); + } + } + + Map paramsTmp = new LinkedHashMap<>(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + Object value = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + paramsTmp.put(fieldName, value); + } + } + + if (logger.isDebugEnabled()) { + logger.trace( + "Join table update es index by query whole sql, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + esTemplate.updateByQuery(config, paramsTmp, esFieldData); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 单表简单字段update + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行data数据 + * @param old 单行old数据 + */ + private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map data, + Map old) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + esTemplate.update(mapping, idVal, esFieldData); + } + + /** + * 主表(单表)复杂字段update + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void mainTableUpdate(ESSyncConfig config, Dml dml, Map data, Map old) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table update to es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + Util.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table update to es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + esTemplate.update(mapping, idVal, esFieldData); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 提交批次 + */ + public void commit() { + esTemplate.commit(); + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESBulkRequest.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESBulkRequest.java new file mode 100644 index 00000000..8fd09d58 --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESBulkRequest.java @@ -0,0 +1,43 @@ +package com.alibaba.otter.canal.client.adapter.es.core.support; + +import java.util.Map; + +public interface ESBulkRequest { + + void resetBulk(); + + ESBulkRequest add(ESIndexRequest esIndexRequest); + + ESBulkRequest add(ESUpdateRequest esUpdateRequest); + + ESBulkRequest add(ESDeleteRequest esDeleteRequest); + + int numberOfActions(); + + ESBulkResponse bulk(); + + interface ESIndexRequest { + + ESIndexRequest setSource(Map source); + + ESIndexRequest setRouting(String routing); + } + + interface ESUpdateRequest { + + ESUpdateRequest setDoc(Map source); + + ESUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc); + + ESUpdateRequest setRouting(String routing); + } + + interface ESDeleteRequest { + } + + interface ESBulkResponse { + boolean hasFailures(); + + void processFailBulkResponse(String errorMsg); + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java new file mode 100644 index 00000000..3d97a2f0 --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java @@ -0,0 +1,303 @@ +package com.alibaba.otter.canal.client.adapter.es.core.support; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.SQLException; +import java.util.*; + +import org.apache.commons.codec.binary.Base64; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.support.Util; + +/** + * ES 同步工具同类 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESSyncUtil { + + private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class); + + public static Object convertToEsObj(Object val, String fieldInfo) { + if (val == null) { + return null; + } + if (fieldInfo.startsWith("array:")) { + String separator = fieldInfo.substring("array:".length()).trim(); + String[] values = val.toString().split(separator); + return Arrays.asList(values); + } else if (fieldInfo.startsWith("object")) { + if (val instanceof String){ + return JSON.parse(val.toString()); + } + return JSON.parse(new String((byte[])val)); + } + return null; + } + + /** + * 类型转换为Mapping中对应的类型 + */ + public static Object typeConvert(Object val, String esType) { + if (val == null) { + return null; + } + if (esType == null) { + return val; + } + Object res = null; + if ("integer".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).intValue(); + } else { + res = Integer.parseInt(val.toString()); + } + } else if ("long".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).longValue(); + } else { + res = Long.parseLong(val.toString()); + } + } else if ("short".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).shortValue(); + } else { + res = Short.parseShort(val.toString()); + } + } else if ("byte".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).byteValue(); + } else { + res = Byte.parseByte(val.toString()); + } + } else if ("double".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).doubleValue(); + } else { + res = Double.parseDouble(val.toString()); + } + } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).floatValue(); + } else { + res = Float.parseFloat(val.toString()); + } + } else if ("boolean".equals(esType)) { + if (val instanceof Boolean) { + res = val; + } else if (val instanceof Number) { + int v = ((Number) val).intValue(); + res = v != 0; + } else { + res = Boolean.parseBoolean(val.toString()); + } + } else if ("date".equals(esType)) { + if (val instanceof java.sql.Time) { + DateTime dateTime = new DateTime(((java.sql.Time) val).getTime()); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("HH:mm:ss.SSS"); + } else { + res = dateTime.toString("HH:mm:ss"); + } + } else if (val instanceof java.sql.Timestamp) { + DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime()); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone); + } + } else if (val instanceof java.sql.Date || val instanceof Date) { + DateTime dateTime; + if (val instanceof java.sql.Date) { + dateTime = new DateTime(((java.sql.Date) val).getTime()); + } else { + dateTime = new DateTime(((Date) val).getTime()); + } + if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 + && dateTime.getMillisOfSecond() == 0) { + res = dateTime.toString("yyyy-MM-dd"); + } else { + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone); + } + } + } else if (val instanceof Long) { + DateTime dateTime = new DateTime(((Long) val).longValue()); + if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 + && dateTime.getMillisOfSecond() == 0) { + res = dateTime.toString("yyyy-MM-dd"); + } else if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone); + } + } else if (val instanceof String) { + String v = ((String) val).trim(); + if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' ' + && v.charAt(13) == ':' && v.charAt(16) == ':') { + String dt = v.substring(0, 10) + "T" + v.substring(11); + Date date = Util.parseDate(dt); + if (date != null) { + DateTime dateTime = new DateTime(date); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone); + } + } + } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') { + Date date = Util.parseDate(v); + if (date != null) { + DateTime dateTime = new DateTime(date); + res = dateTime.toString("yyyy-MM-dd"); + } + } + } + } else if ("binary".equals(esType)) { + if (val instanceof byte[]) { + Base64 base64 = new Base64(); + res = base64.encodeAsString((byte[]) val); + } else if (val instanceof Blob) { + byte[] b = blobToBytes((Blob) val); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); + } else if (val instanceof String) { + // 对应canal中的单字节编码 + byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); + } + } else if ("geo_point".equals(esType)) { + if (!(val instanceof String)) { + logger.error("es type is geo_point, but source type is not String"); + return val; + } + + if (!((String) val).contains(",")) { + logger.error("es type is geo_point, source value not contains ',' separator"); + return val; + } + + String[] point = ((String) val).split(","); + Map location = new HashMap<>(); + location.put("lat", Double.valueOf(point[0].trim())); + location.put("lon", Double.valueOf(point[1].trim())); + return location; + } else if ("array".equals(esType)) { + if ("".equals(val.toString().trim())) { + res = new ArrayList<>(); + } else { + String value = val.toString(); + String separator = ","; + if (!value.contains(",")) { + if (value.contains(";")) { + separator = ";"; + } else if (value.contains("|")) { + separator = "\\|"; + } else if (value.contains("-")) { + separator = "-"; + } + } + String[] values = value.split(separator); + return Arrays.asList(values); + } + } else if ("object".equals(esType)) { + if ("".equals(val.toString().trim())) { + res = new HashMap<>(); + } else { + res = JSON.parseObject(val.toString(), Map.class); + } + } else { + // 其他类全以字符串处理 + res = val.toString(); + } + + return res; + } + + /** + * Blob转byte[] + */ + private static byte[] blobToBytes(Blob blob) { + try (InputStream is = blob.getBinaryStream()) { + byte[] b = new byte[(int) blob.length()]; + if (is.read(b) != -1) { + return b; + } else { + return new byte[0]; + } + } catch (IOException | SQLException e) { + logger.error(e.getMessage()); + return null; + } + } + + /** + * 拼接主键条件 + * + * @param mapping + * @param data + * @return + */ + public static String pkConditionSql(ESMapping mapping, Map data) { + Set idColumns = new LinkedHashSet<>(); + SchemaItem schemaItem = mapping.getSchemaItem(); + + TableItem mainTable = schemaItem.getMainTable(); + + for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) { + if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null) + || (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) { + idColumns.add(idColumnItem); + } + } + + if (idColumns.isEmpty()) { + throw new RuntimeException("Not found primary key field in main table"); + } + + // 拼接condition + StringBuilder condition = new StringBuilder(" "); + for (ColumnItem idColumn : idColumns) { + Object idVal = data.get(idColumn.getColumnName()); + if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append("."); + condition.append(idColumn.getColumnName()).append("="); + if (idVal instanceof String) { + condition.append("'").append(idVal).append("' AND "); + } else { + condition.append(idVal).append(" AND "); + } + } + + if (condition.toString().endsWith("AND ")) { + int len2 = condition.length(); + condition.delete(len2 - 4, len2); + } + return condition.toString(); + } + + public static String appendCondition(String sql, String condition) { + return sql + " WHERE " + condition + " "; + } + + public static void appendCondition(StringBuilder sql, Object value, String owner, String columnName) { + if (value instanceof String) { + sql.append(owner).append(".").append(columnName).append("='").append(value).append("' AND "); + } else { + sql.append(owner).append(".").append(columnName).append("=").append(value).append(" AND "); + } + } +} diff --git a/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESTemplate.java b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESTemplate.java new file mode 100644 index 00000000..2286b915 --- /dev/null +++ b/client-adapter/es-core/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESTemplate.java @@ -0,0 +1,69 @@ +package com.alibaba.otter.canal.client.adapter.es.core.support; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; + +public interface ESTemplate { + + /** + * 插入数据 + * + * @param mapping 配置对象 + * @param pkVal 主键值 + * @param esFieldData 数据Map + */ + void insert(ESMapping mapping, Object pkVal, Map esFieldData); + + /** + * 根据主键更新数据 + * + * @param mapping 配置对象 + * @param pkVal 主键值 + * @param esFieldData 数据Map + */ + void update(ESMapping mapping, Object pkVal, Map esFieldData); + + /** + * update by query + * + * @param config 配置对象 + * @param paramsTmp sql查询条件 + * @param esFieldData 数据Map + */ + void updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData); + + /** + * 通过主键删除数据 + * + * @param mapping 配置对象 + * @param pkVal 主键值 + * @param esFieldData 数据Map + */ + void delete(ESMapping mapping, Object pkVal, Map esFieldData); + + /** + * 提交批次 + */ + void commit(); + + Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException; + + Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map esFieldData) throws SQLException; + + Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException; + + Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException; + + Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName); + + Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map esFieldData); + + Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, + Map esFieldData); +} diff --git a/client-adapter/es6x/pom.xml b/client-adapter/es6x/pom.xml new file mode 100644 index 00000000..1056b71a --- /dev/null +++ b/client-adapter/es6x/pom.xml @@ -0,0 +1,101 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.5-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.es6x + jar + canal client adapter es v6x module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.common + ${project.version} + provided + + + com.alibaba.otter + client-adapter.es-core + ${project.version} + + + org.elasticsearch + elasticsearch + 6.4.3 + + + org.elasticsearch.client + transport + 6.4.3 + + + org.elasticsearch.client + elasticsearch-rest-client + 6.4.3 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.4.3 + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + + diff --git a/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java new file mode 100644 index 00000000..1ffa0c2d --- /dev/null +++ b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java @@ -0,0 +1,123 @@ +package com.alibaba.otter.canal.client.adapter.es6x; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.etl.ESEtlService; +import com.alibaba.otter.canal.client.adapter.es6x.support.ES6xTemplate; +import com.alibaba.otter.canal.client.adapter.es6x.support.ESConnection; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.SPI; +import org.elasticsearch.action.search.SearchResponse; + +import javax.sql.DataSource; + +/** + * ES 6.x 外部适配器 + * + * @author rewerma 2019-09-23 + * @version 1.0.0 + */ +@SPI("es6") +public class ES6xAdapter extends ESAdapter { + + private ESConnection esConnection; + + public ESConnection getEsConnection() { + return esConnection; + } + + @Override + public void init(OuterAdapterConfig configuration, Properties envProperties) { + try { + Map properties = configuration.getProperties(); + + String[] hostArray = configuration.getHosts().split(","); + String mode = properties.get("mode"); + if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) { + esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST); + } else { + esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT); + } + this.esTemplate = new ES6xTemplate(esConnection); + + envProperties.put("es.version", "es6"); + super.init(configuration, envProperties); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public Map count(String task) { + ESSyncConfig config = esSyncConfig.get(task); + ESSyncConfig.ESMapping mapping = config.getEsMapping(); + SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).size(0) + .getResponse(); + + long rowCount = response.getHits().getTotalHits(); + Map res = new LinkedHashMap<>(); + res.put("esIndex", mapping.get_index()); + res.put("count", rowCount); + return res; + } + + @Override + public EtlResult etl(String task, List params) { + EtlResult etlResult = new EtlResult(); + ESSyncConfig config = esSyncConfig.get(task); + if (config != null) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + ESEtlService esEtlService = new ESEtlService(esConnection, config); + if (dataSource != null) { + return esEtlService.importData(params); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } + } else { + StringBuilder resultMsg = new StringBuilder(); + boolean resSuccess = true; + for (ESSyncConfig configTmp : esSyncConfig.values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + ESEtlService esEtlService = new ESEtlService(esConnection, configTmp); + EtlResult etlRes = esEtlService.importData(params); + if (!etlRes.getSucceeded()) { + resSuccess = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSuccess); + if (resSuccess) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; + } + + @Override + public void destroy() { + super.destroy(); + if (esConnection != null) { + esConnection.close(); + } + } +} diff --git a/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java new file mode 100644 index 00000000..1f4320b4 --- /dev/null +++ b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java @@ -0,0 +1,202 @@ +package com.alibaba.otter.canal.client.adapter.es6x.etl; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.es6x.support.ES6xTemplate; +import com.alibaba.otter.canal.client.adapter.es6x.support.ESConnection; +import com.alibaba.otter.canal.client.adapter.es6x.support.ESConnection.ESSearchRequest; +import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.Util; + +/** + * ES ETL Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESEtlService extends AbstractEtlService { + + private ESConnection esConnection; + private ESTemplate esTemplate; + private ESSyncConfig config; + + public ESEtlService(ESConnection esConnection, ESSyncConfig config){ + super("ES", config); + this.esConnection = esConnection; + this.esTemplate = new ES6xTemplate(esConnection); + this.config = config; + } + + public EtlResult importData(List params) { + ESMapping mapping = config.getEsMapping(); + logger.info("start etl to import data to index: {}", mapping.get_index()); + String sql = mapping.getSql(); + return importData(sql, params); + } + + protected boolean executeSqlImport(DataSource ds, String sql, List values, + AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount, + List errMsg) { + try { + ESMapping mapping = (ESMapping) adapterMapping; + Util.sqlRS(ds, sql, values, rs -> { + int count = 0; + try { + ESBulkRequest esBulkRequest = this.esConnection.new ES6xBulkRequest(); + + long batchBegin = System.currentTimeMillis(); + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = null; + for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) { + + String fieldName = fieldItem.getFieldName(); + if (mapping.getSkips().contains(fieldName)) { + continue; + } + + // 如果是主键字段则不插入 + if (fieldItem.getFieldName().equals(mapping.get_id())) { + idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName); + } else { + Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName); + esFieldData.put(Util.cleanColumn(fieldName), val); + } + + } + + if (!mapping.getRelations().isEmpty()) { + mapping.getRelations().forEach((relationField, relationMapping) -> { + Map relations = new HashMap<>(); + relations.put("name", relationMapping.getName()); + if (StringUtils.isNotEmpty(relationMapping.getParent())) { + FieldItem parentFieldItem = mapping.getSchemaItem() + .getSelectFields() + .get(relationMapping.getParent()); + Object parentVal; + try { + parentVal = esTemplate.getValFromRS(mapping, + rs, + parentFieldItem.getFieldName(), + parentFieldItem.getFieldName()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + if (parentVal != null) { + relations.put("parent", parentVal.toString()); + esFieldData.put("$parent_routing", parentVal.toString()); + + } + } + esFieldData.put(Util.cleanColumn(relationField), relations); + }); + } + + if (idVal != null) { + String parentVal = (String) esFieldData.remove("$parent_routing"); + if (mapping.isUpsert()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest( + mapping.get_index(), + mapping.get_type(), + idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); + + if (StringUtils.isNotEmpty(parentVal)) { + esUpdateRequest.setRouting(parentVal); + } + + esBulkRequest.add(esUpdateRequest); + } else { + ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping + .get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData); + if (StringUtils.isNotEmpty(parentVal)) { + esIndexRequest.setRouting(parentVal); + } + esBulkRequest.add(esIndexRequest); + } + } else { + idVal = esFieldData.get(mapping.getPk()); + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), + mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)) + .size(10000); + SearchResponse response = esSearchRequest.getResponse(); + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping + .get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData); + esBulkRequest.add(esUpdateRequest); + } + } + + if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 + && esBulkRequest.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + ESBulkResponse rp = esBulkRequest.bulk(); + if (rp.hasFailures()) { + rp.processFailBulkResponse("全量数据 etl 异常 "); + } + + if (logger.isTraceEnabled()) { + logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + esBulkRequest.numberOfActions(), + mapping.get_index()); + } + batchBegin = System.currentTimeMillis(); + esBulkRequest.resetBulk(); + } + count++; + impCount.incrementAndGet(); + } + + if (esBulkRequest.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + ESBulkResponse rp = esBulkRequest.bulk(); + if (rp.hasFailures()) { + rp.processFailBulkResponse("全量数据 etl 异常 "); + } + if (logger.isTraceEnabled()) { + logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + esBulkRequest.numberOfActions(), + mapping.get_index()); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage()); + throw new RuntimeException(e); + } + return count; + }); + + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } +} diff --git a/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java new file mode 100644 index 00000000..d8df0908 --- /dev/null +++ b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java @@ -0,0 +1,468 @@ +package com.alibaba.otter.canal.client.adapter.es6x.support; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.es6x.support.ESConnection.ESSearchRequest; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class ES6xTemplate implements ESTemplate { + + private static final Logger logger = LoggerFactory + .getLogger(ESTemplate.class); + + private static final int MAX_BATCH_SIZE = 1000; + + private ESConnection esConnection; + + private ESBulkRequest esBulkRequest; + + // es 字段类型本地缓存 + private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + + public ES6xTemplate(ESConnection esConnection){ + this.esConnection = esConnection; + this.esBulkRequest = this.esConnection.new ES6xBulkRequest(); + } + + public ESBulkRequest getBulk() { + return esBulkRequest; + } + + public void resetBulkRequestBuilder() { + this.esBulkRequest.resetBulk(); + } + + @Override + public void insert(ESSyncConfig.ESMapping mapping, Object pkVal, Map esFieldData) { + if (mapping.get_id() != null) { + String parentVal = (String) esFieldData.remove("$parent_routing"); + if (mapping.isUpsert()) { + ESUpdateRequest updateRequest = esConnection.new ES6xUpdateRequest(mapping.get_index(), + mapping.get_type(), + pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); + if (StringUtils.isNotEmpty(parentVal)) { + updateRequest.setRouting(parentVal); + } + getBulk().add(updateRequest); + } else { + ESIndexRequest indexRequest = esConnection.new ES6xIndexRequest(mapping.get_index(), + mapping.get_type(), + pkVal.toString()).setSource(esFieldData); + if (StringUtils.isNotEmpty(parentVal)) { + indexRequest.setRouting(parentVal); + } + getBulk().add(indexRequest); + } + commitBulk(); + } else { + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), + mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000); + SearchResponse response = esSearchRequest.getResponse(); + + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), + mapping.get_type(), + hit.getId()).setDoc(esFieldData); + getBulk().add(esUpdateRequest); + commitBulk(); + } + } + } + + @Override + public void update(ESSyncConfig.ESMapping mapping, Object pkVal, Map esFieldData) { + Map esFieldDataTmp = new LinkedHashMap<>(esFieldData.size()); + esFieldData.forEach((k, v) -> esFieldDataTmp.put(Util.cleanColumn(k), v)); + append4Update(mapping, pkVal, esFieldDataTmp); + commitBulk(); + } + + @Override + public void updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { + if (paramsTmp.isEmpty()) { + return; + } + ESMapping mapping = config.getEsMapping(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); + + // 查询sql批量更新 + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); + List values = new ArrayList<>(); + paramsTmp.forEach((fieldName, value) -> { + sql.append("_v.").append(fieldName).append("=? AND "); + values.add(value); + }); + // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件 + int len = sql.length(); + sql.delete(len - 4, len); + Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> { + int count = 0; + try { + while (rs.next()) { + Object idVal = getIdValFromRS(mapping, rs); + append4Update(mapping, idVal, esFieldData); + commitBulk(); + count++; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return count; + }); + if (logger.isTraceEnabled()) { + logger.trace("Update ES by query affected {} records", syncCount); + } + } + + @Override + public void delete(ESSyncConfig.ESMapping mapping, Object pkVal, Map esFieldData) { + if (mapping.get_id() != null) { + ESDeleteRequest esDeleteRequest = this.esConnection.new ES6xDeleteRequest(mapping.get_index(), + mapping.get_type(), + pkVal.toString()); + getBulk().add(esDeleteRequest); + commitBulk(); + } else { + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), + mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000); + SearchResponse response = esSearchRequest.getResponse(); + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), + mapping.get_type(), + hit.getId()).setDoc(esFieldData); + getBulk().add(esUpdateRequest); + commitBulk(); + } + } + } + + @Override + public void commit() { + if (getBulk().numberOfActions() > 0) { + ESBulkResponse response = getBulk().bulk(); + if (response.hasFailures()) { + response.processFailBulkResponse("ES sync commit error "); + } + resetBulkRequestBuilder(); + } + } + + @Override + public Object getValFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException { + fieldName = Util.cleanColumn(fieldName); + columnName = Util.cleanColumn(columnName); + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(columnName); + if (value instanceof Boolean) { + if (!"boolean".equals(esType)) { + value = resultSet.getByte(columnName); + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + @Override + public Object getESDataFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); + } + } + + // 添加父子文档关联信息 + putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData); + + return resultIdVal; + } + + @Override + public Object getIdValFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + break; + } + } + return resultIdVal; + } + + @Override + public Object getESDataFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + } + + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (dmlOld.containsKey(columnItem.getColumnName()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), + getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); + break; + } + } + } + + // 添加父子文档关联信息 + putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData); + + return resultIdVal; + } + + @Override + public Object getValFromData(ESSyncConfig.ESMapping mapping, Map dmlData, String fieldName, + String columnName) { + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(columnName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + @Override + public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping, Map dmlData, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); + } + } + + // 添加父子文档关联信息 + putRelationData(mapping, schemaItem, dmlData, esFieldData); + return resultIdVal; + } + + @Override + public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping, Map dmlData, + Map dmlOld, Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + } + + if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), + getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); + } + } + + // 添加父子文档关联信息 + putRelationData(mapping, schemaItem, dmlOld, esFieldData); + return resultIdVal; + } + + /** + * 如果大于批量数则提交批次 + */ + private void commitBulk() { + if (getBulk().numberOfActions() >= MAX_BATCH_SIZE) { + commit(); + } + } + + private void append4Update(ESMapping mapping, Object pkVal, Map esFieldData) { + if (mapping.get_id() != null) { + String parentVal = (String) esFieldData.remove("$parent_routing"); + if (mapping.isUpsert()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), + mapping.get_type(), + pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); + if (StringUtils.isNotEmpty(parentVal)) { + esUpdateRequest.setRouting(parentVal); + } + getBulk().add(esUpdateRequest); + } else { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), + mapping.get_type(), + pkVal.toString()).setDoc(esFieldData); + if (StringUtils.isNotEmpty(parentVal)) { + esUpdateRequest.setRouting(parentVal); + } + getBulk().add(esUpdateRequest); + } + } else { + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), + mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000); + SearchResponse response = esSearchRequest.getResponse(); + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), + mapping.get_type(), + hit.getId()).setDoc(esFieldData); + getBulk().add(esUpdateRequest); + } + } + } + + /** + * 获取es mapping中的属性类型 + * + * @param mapping mapping配置 + * @param fieldName 属性名 + * @return 类型 + */ + @SuppressWarnings("unchecked") + private String getEsType(ESMapping mapping, String fieldName) { + String key = mapping.get_index() + "-" + mapping.get_type(); + Map fieldType = esFieldTypes.get(key); + if (fieldType != null) { + return fieldType.get(fieldName); + } else { + MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index(), mapping.get_type()); + + if (mappingMetaData == null) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + + fieldType = new LinkedHashMap<>(); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); + if (value.containsKey("properties")) { + fieldType.put(entry.getKey(), "object"); + } else { + fieldType.put(entry.getKey(), (String) value.get("type")); + } + } + esFieldTypes.put(key, fieldType); + + return fieldType.get(fieldName); + } + } + + private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet, + Map esFieldData) { + // 添加父子文档关联信息 + if (!mapping.getRelations().isEmpty()) { + mapping.getRelations().forEach((relationField, relationMapping) -> { + Map relations = new HashMap<>(); + relations.put("name", relationMapping.getName()); + if (StringUtils.isNotEmpty(relationMapping.getParent())) { + FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent()); + Object parentVal; + try { + parentVal = getValFromRS(mapping, + resultSet, + parentFieldItem.getFieldName(), + parentFieldItem.getFieldName()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + if (parentVal != null) { + relations.put("parent", parentVal.toString()); + esFieldData.put("$parent_routing", parentVal.toString()); + + } + } + esFieldData.put(relationField, relations); + }); + } + } + + private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map dmlData, + Map esFieldData) { + // 添加父子文档关联信息 + if (!mapping.getRelations().isEmpty()) { + mapping.getRelations().forEach((relationField, relationMapping) -> { + Map relations = new HashMap<>(); + relations.put("name", relationMapping.getName()); + if (StringUtils.isNotEmpty(relationMapping.getParent())) { + FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent()); + String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName(); + Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName); + if (parentVal != null) { + relations.put("parent", parentVal.toString()); + esFieldData.put("$parent_routing", parentVal.toString()); + + } + } + esFieldData.put(relationField, relations); + }); + } + } +} diff --git a/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java new file mode 100644 index 00000000..88c34ba6 --- /dev/null +++ b/client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java @@ -0,0 +1,505 @@ +package com.alibaba.otter.canal.client.adapter.es6x.support; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.elasticsearch.client.*; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * ES 连接器, Transport Rest 两种方式 + * + * @author rewerma 2019-08-01 + * @version 1.0.0 + */ +public class ESConnection { + + private static final Logger logger = LoggerFactory.getLogger(ESConnection.class); + + public enum ESClientMode { + TRANSPORT, REST + } + + private ESClientMode mode; + + private TransportClient transportClient; + + private RestHighLevelClient restHighLevelClient; + + public ESConnection(String[] hosts, Map properties, ESClientMode mode) throws UnknownHostException{ + this.mode = mode; + if (mode == ESClientMode.TRANSPORT) { + Settings.Builder settingBuilder = Settings.builder(); + settingBuilder.put("cluster.name", properties.get("cluster.name")); + Settings settings = settingBuilder.build(); + transportClient = new PreBuiltTransportClient(settings); + for (String host : hosts) { + int i = host.indexOf(":"); + transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)), + Integer.parseInt(host.substring(i + 1)))); + } + } else { + HttpHost[] httpHosts = new HttpHost[hosts.length]; + for (int i = 0; i < hosts.length; i++) { + String host = hosts[i]; + int j = host.indexOf(":"); + HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)), + Integer.parseInt(host.substring(j + 1))); + httpHosts[i] = httpHost; + } + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + String nameAndPwd = properties.get("security.auth"); + if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) { + String[] nameAndPwdArr = nameAndPwd.split(":"); + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1])); + restClientBuilder.setHttpClientConfigCallback( + httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + restHighLevelClient = new RestHighLevelClient(restClientBuilder); + } + } + + public void close() { + if (mode == ESClientMode.TRANSPORT) { + transportClient.close(); + } else { + try { + restHighLevelClient.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public MappingMetaData getMapping(String index, String type) { + MappingMetaData mappingMetaData = null; + if (mode == ESClientMode.TRANSPORT) { + ImmutableOpenMap mappings; + try { + mappings = transportClient.admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices() + .get(index) + .getMappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + index); + } + mappingMetaData = mappings.get(type); + + } else { + ImmutableOpenMap> mappings; + try { + GetMappingsRequest request = new GetMappingsRequest(); + request.indices(index); + GetMappingsResponse response; + // try { + // response = restHighLevelClient + // .indices() + // .getMapping(request, RequestOptions.DEFAULT); + // // 6.4以下版本直接使用该接口会报错 + // } catch (Exception e) { + // logger.warn("Low ElasticSearch version for getMapping"); + response = RestHighLevelClientExt.getMapping(restHighLevelClient, request, RequestOptions.DEFAULT); + // } + + mappings = response.mappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + index); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return null; + } + mappingMetaData = mappings.get(index).get(type); + } + return mappingMetaData; + } + + public class ES6xIndexRequest implements ESBulkRequest.ESIndexRequest { + + private IndexRequestBuilder indexRequestBuilder; + + private IndexRequest indexRequest; + + public ES6xIndexRequest(String index, String type, String id){ + if (mode == ESClientMode.TRANSPORT) { + indexRequestBuilder = transportClient.prepareIndex(index, type, id); + } else { + indexRequest = new IndexRequest(index, type, id); + } + } + + public ES6xIndexRequest setSource(Map source) { + if (mode == ESClientMode.TRANSPORT) { + indexRequestBuilder.setSource(source); + } else { + indexRequest.source(source); + } + return this; + } + + public ES6xIndexRequest setRouting(String routing) { + if (mode == ESClientMode.TRANSPORT) { + indexRequestBuilder.setRouting(routing); + } else { + indexRequest.routing(routing); + } + return this; + } + + public IndexRequestBuilder getIndexRequestBuilder() { + return indexRequestBuilder; + } + + public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) { + this.indexRequestBuilder = indexRequestBuilder; + } + + public IndexRequest getIndexRequest() { + return indexRequest; + } + + public void setIndexRequest(IndexRequest indexRequest) { + this.indexRequest = indexRequest; + } + } + + public class ES6xUpdateRequest implements ESBulkRequest.ESUpdateRequest { + + private UpdateRequestBuilder updateRequestBuilder; + + private UpdateRequest updateRequest; + + public ES6xUpdateRequest(String index, String type, String id){ + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder = transportClient.prepareUpdate(index, type, id); + } else { + updateRequest = new UpdateRequest(index, type, id); + } + } + + public ES6xUpdateRequest setDoc(Map source) { + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder.setDoc(source); + } else { + updateRequest.doc(source); + } + return this; + } + + public ES6xUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) { + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder.setDocAsUpsert(shouldUpsertDoc); + } else { + updateRequest.docAsUpsert(shouldUpsertDoc); + } + return this; + } + + public ES6xUpdateRequest setRouting(String routing) { + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder.setRouting(routing); + } else { + updateRequest.routing(routing); + } + return this; + } + + public UpdateRequestBuilder getUpdateRequestBuilder() { + return updateRequestBuilder; + } + + public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) { + this.updateRequestBuilder = updateRequestBuilder; + } + + public UpdateRequest getUpdateRequest() { + return updateRequest; + } + + public void setUpdateRequest(UpdateRequest updateRequest) { + this.updateRequest = updateRequest; + } + } + + public class ES6xDeleteRequest implements ESBulkRequest.ESDeleteRequest { + + private DeleteRequestBuilder deleteRequestBuilder; + + private DeleteRequest deleteRequest; + + public ES6xDeleteRequest(String index, String type, String id){ + if (mode == ESClientMode.TRANSPORT) { + deleteRequestBuilder = transportClient.prepareDelete(index, type, id); + } else { + deleteRequest = new DeleteRequest(index, type, id); + } + } + + public DeleteRequestBuilder getDeleteRequestBuilder() { + return deleteRequestBuilder; + } + + public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) { + this.deleteRequestBuilder = deleteRequestBuilder; + } + + public DeleteRequest getDeleteRequest() { + return deleteRequest; + } + + public void setDeleteRequest(DeleteRequest deleteRequest) { + this.deleteRequest = deleteRequest; + } + } + + public class ESSearchRequest { + + private SearchRequestBuilder searchRequestBuilder; + + private SearchRequest searchRequest; + + private SearchSourceBuilder sourceBuilder; + + public ESSearchRequest(String index, String... types){ + if (mode == ESClientMode.TRANSPORT) { + searchRequestBuilder = transportClient.prepareSearch(index).setTypes(types); + } else { + searchRequest = new SearchRequest(index).types(types); + sourceBuilder = new SearchSourceBuilder(); + } + } + + public ESSearchRequest setQuery(QueryBuilder queryBuilder) { + if (mode == ESClientMode.TRANSPORT) { + searchRequestBuilder.setQuery(queryBuilder); + } else { + sourceBuilder.query(queryBuilder); + } + return this; + } + + public ESSearchRequest size(int size) { + if (mode == ESClientMode.TRANSPORT) { + searchRequestBuilder.setSize(size); + } else { + sourceBuilder.size(size); + } + return this; + } + + public SearchResponse getResponse() { + if (mode == ESClientMode.TRANSPORT) { + return searchRequestBuilder.get(); + } else { + searchRequest.source(sourceBuilder); + try { + return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public SearchRequestBuilder getSearchRequestBuilder() { + return searchRequestBuilder; + } + + public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) { + this.searchRequestBuilder = searchRequestBuilder; + } + + public SearchRequest getSearchRequest() { + return searchRequest; + } + + public void setSearchRequest(SearchRequest searchRequest) { + this.searchRequest = searchRequest; + } + } + + public class ES6xBulkRequest implements ESBulkRequest { + + private BulkRequestBuilder bulkRequestBuilder; + + private BulkRequest bulkRequest; + + public ES6xBulkRequest(){ + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder = transportClient.prepareBulk(); + } else { + bulkRequest = new BulkRequest(); + } + } + + public void resetBulk() { + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder = transportClient.prepareBulk(); + } else { + bulkRequest = new BulkRequest(); + } + } + + public ES6xBulkRequest add(ESIndexRequest esIndexRequest) { + ES6xIndexRequest eir = (ES6xIndexRequest) esIndexRequest; + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder.add(eir.indexRequestBuilder); + } else { + bulkRequest.add(eir.indexRequest); + } + return this; + } + + public ES6xBulkRequest add(ESUpdateRequest esUpdateRequest) { + ES6xUpdateRequest eur = (ES6xUpdateRequest) esUpdateRequest; + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder.add(eur.updateRequestBuilder); + } else { + bulkRequest.add(eur.updateRequest); + } + return this; + } + + public ES6xBulkRequest add(ESDeleteRequest esDeleteRequest) { + ES6xDeleteRequest edr = (ES6xDeleteRequest) esDeleteRequest; + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder.add(edr.deleteRequestBuilder); + } else { + bulkRequest.add(edr.deleteRequest); + } + return this; + } + + public int numberOfActions() { + if (mode == ESClientMode.TRANSPORT) { + return bulkRequestBuilder.numberOfActions(); + } else { + return bulkRequest.numberOfActions(); + } + } + + public ESBulkResponse bulk() { + if (mode == ESClientMode.TRANSPORT) { + BulkResponse responses = bulkRequestBuilder.execute().actionGet(); + return new ES6xBulkResponse(responses); + } else { + try { + BulkResponse responses = restHighLevelClient.bulk(bulkRequest); + return new ES6xBulkResponse(responses); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public BulkRequestBuilder getBulkRequestBuilder() { + return bulkRequestBuilder; + } + + public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) { + this.bulkRequestBuilder = bulkRequestBuilder; + } + + public BulkRequest getBulkRequest() { + return bulkRequest; + } + + public void setBulkRequest(BulkRequest bulkRequest) { + this.bulkRequest = bulkRequest; + } + } + + public static class ES6xBulkResponse implements ESBulkRequest.ESBulkResponse { + + private BulkResponse bulkResponse; + + public ES6xBulkResponse(BulkResponse bulkResponse){ + this.bulkResponse = bulkResponse; + } + + @Override + public boolean hasFailures() { + return bulkResponse.hasFailures(); + } + + @Override + public void processFailBulkResponse(String errorMsg) { + for (BulkItemResponse itemResponse : bulkResponse.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.error(itemResponse.getFailureMessage()); + } else { + throw new RuntimeException(errorMsg + itemResponse.getFailureMessage()); + } + } + } + } + + // ------ get/set ------ + public ESClientMode getMode() { + return mode; + } + + public void setMode(ESClientMode mode) { + this.mode = mode; + } + + public TransportClient getTransportClient() { + return transportClient; + } + + public void setTransportClient(TransportClient transportClient) { + this.transportClient = transportClient; + } + + public RestHighLevelClient getRestHighLevelClient() { + return restHighLevelClient; + } + + public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + } +} diff --git a/client-adapter/es6x/src/main/java/org/elasticsearch/client/RequestConvertersExt.java b/client-adapter/es6x/src/main/java/org/elasticsearch/client/RequestConvertersExt.java new file mode 100644 index 00000000..50ff32fa --- /dev/null +++ b/client-adapter/es6x/src/main/java/org/elasticsearch/client/RequestConvertersExt.java @@ -0,0 +1,30 @@ +package org.elasticsearch.client; + +import java.io.IOException; + +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.common.Strings; + +/** + * RequestConverters扩展 + * + * @author rewerma 2019-08-01 + * @version 1.0.0 + */ +public class RequestConvertersExt { + + /** + * 修改 getMappings 去掉request参数 + * + * @param getMappingsRequest + * @return + * @throws IOException + */ + static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOException { + String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices(); + String[] types = getMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.types(); + + return new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping", types)); + } +} diff --git a/client-adapter/es6x/src/main/java/org/elasticsearch/client/RestHighLevelClientExt.java b/client-adapter/es6x/src/main/java/org/elasticsearch/client/RestHighLevelClientExt.java new file mode 100644 index 00000000..a55295ef --- /dev/null +++ b/client-adapter/es6x/src/main/java/org/elasticsearch/client/RestHighLevelClientExt.java @@ -0,0 +1,29 @@ +package org.elasticsearch.client; + +import static java.util.Collections.emptySet; + +import java.io.IOException; + +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; + +/** + * RestHighLevelClient扩展 + * + * @author rewerma 2019-08-01 + * @version 1.0.0 + */ +public class RestHighLevelClientExt { + + public static GetMappingsResponse getMapping(RestHighLevelClient restHighLevelClient, + GetMappingsRequest getMappingsRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest, + RequestConvertersExt::getMappings, + options, + GetMappingsResponse::fromXContent, + emptySet()); + + } + +} diff --git a/client-adapter/es6x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter b/client-adapter/es6x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter new file mode 100644 index 00000000..fa5e19a4 --- /dev/null +++ b/client-adapter/es6x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter @@ -0,0 +1 @@ +es6=com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter diff --git a/client-adapter/es6x/src/main/resources/es6/biz_order.yml b/client-adapter/es6x/src/main/resources/es6/biz_order.yml new file mode 100644 index 00000000..98b9880d --- /dev/null +++ b/client-adapter/es6x/src/main/resources/es6/biz_order.yml @@ -0,0 +1,21 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: customer + _type: _doc + _id: _id + relations: + customer_order: + name: order + parent: customer_id + sql: "select concat('oid_', t.id) as _id, + t.customer_id, + t.id as order_id, + t.serial_code as order_serial, + t.c_time as order_time + from biz_order t" + skips: + - customer_id + etlCondition: "where t.c_time>={}" + commitBatch: 3000 diff --git a/client-adapter/es6x/src/main/resources/es6/customer.yml b/client-adapter/es6x/src/main/resources/es6/customer.yml new file mode 100644 index 00000000..1162fdf0 --- /dev/null +++ b/client-adapter/es6x/src/main/resources/es6/customer.yml @@ -0,0 +1,47 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: customer + _type: _doc + _id: id + relations: + customer_order: + name: customer + sql: "select t.id, t.name, t.email from customer t" + etlCondition: "where t.c_time>={}" + commitBatch: 3000 + + +#{ +# "mappings":{ +# "_doc":{ +# "properties":{ +# "id": { +# "type": "long" +# }, +# "name": { +# "type": "text" +# }, +# "email": { +# "type": "text" +# }, +# "order_id": { +# "type": "long" +# }, +# "order_serial": { +# "type": "text" +# }, +# "order_time": { +# "type": "date" +# }, +# "customer_order":{ +# "type":"join", +# "relations":{ +# "customer":"order" +# } +# } +# } +# } +# } +#} diff --git a/client-adapter/es6x/src/main/resources/es6/mytest_user.yml b/client-adapter/es6x/src/main/resources/es6/mytest_user.yml new file mode 100644 index 00000000..8c2828a0 --- /dev/null +++ b/client-adapter/es6x/src/main/resources/es6/mytest_user.yml @@ -0,0 +1,16 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: mytest_user + _type: _doc + _id: _id + upsert: true +# pk: id + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, + a.c_time as _c_time from user a + left join role b on b.id=a.role_id" +# objFields: +# _labels: array:; + etlCondition: "where a.c_time>={}" + commitBatch: 3000 diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ConfigLoadTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ConfigLoadTest.java new file mode 100644 index 00000000..2d5c74b6 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ConfigLoadTest.java @@ -0,0 +1,41 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test; + +import java.util.Map; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +@Ignore +public class ConfigLoadTest { + + @Before + public void before() { + // AdapterConfigs.put("es", "mytest_user.yml"); + // 加载数据源连接池 + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + } + + @Test + public void testLoad() { + Map configMap = ESSyncConfigLoader.load(null); + ESSyncConfig config = configMap.get("mytest_user.yml"); + config.validate(); + Assert.assertNotNull(config); + Assert.assertEquals("defaultDS", config.getDataSourceKey()); + ESSyncConfig.ESMapping esMapping = config.getEsMapping(); + Assert.assertEquals("mytest_user", esMapping.get_index()); + Assert.assertEquals("_doc", esMapping.get_type()); + Assert.assertEquals("id", esMapping.get_id()); + Assert.assertNotNull(esMapping.getSql()); + + // Map> dbTableEsSyncConfig = + // ESSyncConfigLoader.getDbTableEsSyncConfig(); + // Assert.assertFalse(dbTableEsSyncConfig.isEmpty()); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ESTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ESTest.java new file mode 100644 index 00000000..c04c5b23 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ESTest.java @@ -0,0 +1,120 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class ESTest { + + private TransportClient transportClient; + + @Before + public void init() throws UnknownHostException { + Settings.Builder settingBuilder = Settings.builder(); + settingBuilder.put("cluster.name", TestConstant.clusterName); + Settings settings = settingBuilder.build(); + transportClient = new PreBuiltTransportClient(settings); + String[] hostArray = TestConstant.esHosts.split(","); + for (String host : hostArray) { + int i = host.indexOf(":"); + transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)), + Integer.parseInt(host.substring(i + 1)))); + } + } + + @Test + public void test01() { + SearchResponse response = transportClient.prepareSearch("test") + .setTypes("osm") + .setQuery(QueryBuilders.termQuery("_id", "1")) + .setSize(10000) + .get(); + for (SearchHit hit : response.getHits()) { + System.out.println(hit.getSourceAsMap().get("data").getClass()); + } + } + + @Test + public void test02() { + Map esFieldData = new LinkedHashMap<>(); + esFieldData.put("userId", 2L); + esFieldData.put("eventId", 4L); + esFieldData.put("eventName", "网络异常"); + esFieldData.put("description", "第四个事件信息"); + + Map relations = new LinkedHashMap<>(); + esFieldData.put("user_event", relations); + relations.put("name", "event"); + relations.put("parent", "2"); + + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + bulkRequestBuilder + .add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData)); + commit(bulkRequestBuilder); + } + + @Test + public void test03() { + Map esFieldData = new LinkedHashMap<>(); + esFieldData.put("userId", 2L); + esFieldData.put("eventName", "网络异常1"); + + Map relations = new LinkedHashMap<>(); + esFieldData.put("user_event", relations); + relations.put("name", "event"); + relations.put("parent", "2"); + + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + bulkRequestBuilder.add(transportClient.prepareUpdate("test", "osm", "2_4").setRouting("2").setDoc(esFieldData)); + commit(bulkRequestBuilder); + } + + @Test + public void test04() { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + bulkRequestBuilder.add(transportClient.prepareDelete("test", "osm", "2_4")); + commit(bulkRequestBuilder); + } + + private void commit(BulkRequestBuilder bulkRequestBuilder) { + if (bulkRequestBuilder.numberOfActions() > 0) { + BulkResponse response = bulkRequestBuilder.execute().actionGet(); + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + System.out.println(itemResponse.getFailureMessage()); + } else { + System.out.println("ES bulk commit error" + itemResponse.getFailureMessage()); + } + } + } + } + } + + @After + public void after() { + transportClient.close(); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/SqlParseTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/SqlParseTest.java new file mode 100644 index 00000000..96e22ff6 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/SqlParseTest.java @@ -0,0 +1,48 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test; + +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser; + +public class SqlParseTest { + + @Test + public void parseTest() { + String sql = "select a.id, CASE WHEN a.id <= 500 THEN '1' else '2' end as id2, " + + "concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a " + + "left join role b on a.role_id=b.id " + + "left join (select user_id, group_concat(label,',') as labels from user_label " + + "group by user_id) c on c.user_id=a.id"; + SchemaItem schemaItem = SqlParser.parse(sql); + + // 通过表名找 TableItem + List tableItems = schemaItem.getTableItemAliases().get("user_label".toLowerCase()); + tableItems.forEach(tableItem -> Assert.assertEquals("c", tableItem.getAlias())); + + TableItem tableItem = tableItems.get(0); + Assert.assertFalse(tableItem.isMain()); + Assert.assertTrue(tableItem.isSubQuery()); + // 通过字段名找 FieldItem + List fieldItems = schemaItem.getColumnFields().get(tableItem.getAlias() + ".labels".toLowerCase()); + fieldItems.forEach( + fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName())); + + // 获取当前表关联条件字段 + Map> relationTableFields = tableItem.getRelationTableFields(); + relationTableFields.keySet() + .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); + + // 获取关联字段在select中的对应字段 + // List relationSelectFieldItem = + // tableItem.getRelationKeyFieldItems(); + // relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", + // fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/TestConstant.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/TestConstant.java new file mode 100644 index 00000000..523487ca --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/TestConstant.java @@ -0,0 +1,40 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test; + +import java.sql.SQLException; + +import com.alibaba.druid.pool.DruidDataSource; + +public class TestConstant { + + public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"; + public final static String jdbcUser = "root"; + public final static String jdbcPassword = "121212"; + + public final static String esHosts = "127.0.0.1:9300"; + public final static String clusterName = "elasticsearch"; + + public final static DruidDataSource dataSource; + + static { + dataSource = new DruidDataSource(); + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUser); + dataSource.setPassword(jdbcPassword); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(1); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setPoolPreparedStatements(false); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + dataSource.setValidationQuery("select 1"); + try { + dataSource.init(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/Common.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/Common.java new file mode 100644 index 00000000..4f11b461 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/Common.java @@ -0,0 +1,68 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import javax.sql.DataSource; + +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.es6x.test.TestConstant; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +public class Common { + + public static ES6xAdapter init() { + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + + OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); + outerAdapterConfig.setName("es"); + outerAdapterConfig.setHosts(TestConstant.esHosts); + Map properties = new HashMap<>(); + properties.put("cluster.name", TestConstant.clusterName); + outerAdapterConfig.setProperties(properties); + + ES6xAdapter esAdapter = new ES6xAdapter(); + esAdapter.init(outerAdapterConfig, null); + return esAdapter; + } + + public static void sqlExe(DataSource dataSource, String sql) { + Connection conn = null; + Statement stmt = null; + try { + conn = dataSource.getConnection(); + conn.setAutoCommit(false); + stmt = conn.createStatement(); + stmt.execute(sql); + conn.commit(); + } catch (Exception e) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e1) { + // ignore + } + } + e.printStackTrace(); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // ignore + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // ignore + } + } + } + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java new file mode 100644 index 00000000..21758163 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java @@ -0,0 +1,141 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +@Ignore +public class LabelSyncJoinSub2Test { + + private ES6xAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_sub2.yml"); + esAdapter = Common.init(); + } + + /** + * 带函数子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1 or id=2"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 2L); + data.put("user_id", 1L); + data.put("label", "b"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("b;a_", response.getSource().get("_labels")); + } + + /** + * 带函数子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update label set label='aa' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "aa"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("label", "v"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("b;aa_", response.getSource().get("_labels")); + } + + /** + * 带函数子查询从表删除 + */ + @Test + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "a"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("b_", response.getSource().get("_labels")); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java new file mode 100644 index 00000000..cf3d4aad --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java @@ -0,0 +1,141 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +@Ignore +public class LabelSyncJoinSubTest { + + private ES6xAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_sub.yml"); + esAdapter = Common.init(); + } + + /** + * 子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1 or id=2"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')"); + Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 2L); + data.put("user_id", 1L); + data.put("label", "b"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("b;a", response.getSource().get("_labels")); + } + + /** + * 子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update label set label='aa' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "aa"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("label", "a"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("b;aa", response.getSource().get("_labels")); + } + + /** + * 子查询从表删除 + */ + @Test + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from label where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id", 1L); + data.put("label", "a"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("b", response.getSource().get("_labels")); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java new file mode 100644 index 00000000..ba87a651 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java @@ -0,0 +1,103 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +@Ignore +public class RoleSyncJoinOne2Test { + + private ES6xAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_one2.yml"); + esAdapter = Common.init(); + } + + /** + * 带函数非子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("admin_", response.getSource().get("_role_name")); + } + + /** + * 带函数非子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update role set role_name='admin3' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin3"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_name", "admin"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("admin3_", response.getSource().get("_role_name")); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java new file mode 100644 index 00000000..5b7eb3bc --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java @@ -0,0 +1,208 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +@Ignore +public class RoleSyncJoinOneTest { + + private ES6xAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_one.yml"); + esAdapter = Common.init(); + } + + /** + * 非子查询从表插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("admin", response.getSource().get("_role_name")); + } + + /** + * 非子查询从表更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update role set role_name='admin2' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin2"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_name", "admin"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("admin2", response.getSource().get("_role_name")); + } + + /** + * 主表更新外键值 + */ + @Test + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=2"); + Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')"); + Common.sqlExe(ds, "update user set role_id=2 where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_id", 2L); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_id", 1L); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("operator", response.getSource().get("_role_name")); + + Common.sqlExe(ds, "update user set role_id=1 where id=1"); + + Dml dml2 = new Dml(); + dml2.setDestination("example"); + dml2.setTs(new Date().getTime()); + dml2.setType("UPDATE"); + dml2.setDatabase("mytest"); + dml2.setTable("user"); + List> dataList2 = new ArrayList<>(); + Map data2 = new LinkedHashMap<>(); + dataList2.add(data2); + data2.put("id", 1L); + data2.put("role_id", 1L); + dml2.setData(dataList2); + List> oldList2 = new ArrayList<>(); + Map old2 = new LinkedHashMap<>(); + oldList2.add(old2); + old2.put("role_id", 2L); + dml2.setOld(oldList2); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2); + + GetResponse response2 = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("admin2", response2.getSource().get("_role_name")); + } + + /** + * 非子查询从表删除 + */ + @Test + public void test04() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertNull(response.getSource().get("_role_name")); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java new file mode 100644 index 00000000..eb4aa5f3 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java @@ -0,0 +1,104 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.util.*; + +import javax.sql.DataSource; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +@Ignore +public class UserSyncJoinOneTest { + + private ES6xAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_join_one.yml"); + esAdapter = Common.init(); + } + + /** + * 主表带函数插入 + */ + @Test + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from user where id=1"); + Common.sqlExe(ds, "insert into user (id,name,role_id) values (1,'Eric',1)"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("Eric_", response.getSource().get("_name")); + } + + /** + * 主表带函数更新 + */ + @Test + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update user set name='Eric2' where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric2"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("Eric2_", response.getSource().get("_name")); + } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java new file mode 100644 index 00000000..147bc9f3 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java @@ -0,0 +1,133 @@ +package com.alibaba.otter.canal.client.adapter.es6x.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es6x.ES6xAdapter; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +@Ignore +public class UserSyncSingleTest { + + private ES6xAdapter esAdapter; + + @Before + public void init() { + // AdapterConfigs.put("es", "mytest_user_single.yml"); + esAdapter = Common.init(); + } + + /** + * 单表插入 + */ + @Test + public void test01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("Eric", response.getSource().get("_name")); + } + + /** + * 单表更新 + */ + @Test + public void test02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric2"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric"); + dml.setOld(oldList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertEquals("Eric2", response.getSource().get("_name")); + } + + /** + * 单表删除 + */ + @Test + public void test03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + dml.setData(dataList); + + String database = dml.getDatabase(); + String table = dml.getTable(); + Map esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table); + + esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml); + + GetResponse response = esAdapter.getEsConnection() + .getTransportClient() + .prepareGet("mytest_user", "_doc", "1") + .get(); + Assert.assertNull(response.getSource()); + } + + // @After + // public void after() { + // esAdapter.destroy(); + // DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + // } +} diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/db_schema.sql b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/db_schema.sql new file mode 100644 index 00000000..c7274ca2 --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/db_schema.sql @@ -0,0 +1,39 @@ +-- ---------------------------- +-- Table structure for label +-- ---------------------------- +DROP TABLE IF EXISTS `label`; +CREATE TABLE `label` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `user_id` bigint(20) NOT NULL, + `label` varchar(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Table structure for role +-- ---------------------------- +DROP TABLE IF EXISTS `role`; +CREATE TABLE `role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `role_name` varchar(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Table structure for user +-- ---------------------------- +DROP TABLE IF EXISTS `user`; +CREATE TABLE `user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(30) NOT NULL, + `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `role_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4; + +insert into user (id,name,role_id) values (1,'Eric',1); +insert into role (id,role_name) values (1,'admin'); +insert into role (id,role_name) values (2,'operator'); +insert into label (id,user_id,label) values (1,1,'a'); +insert into label (id,user_id,label) values (2,1,'b'); +commit; \ No newline at end of file diff --git a/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/es_mapping.json b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/es_mapping.json new file mode 100644 index 00000000..55b0389a --- /dev/null +++ b/client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/es_mapping.json @@ -0,0 +1,21 @@ +{ + "_doc": { + "properties": { + "_name": { + "type": "text" + }, + "_role_id": { + "type": "long" + }, + "_role_name": { + "type": "text" + }, + "_labels": { + "type": "text" + }, + "_c_time": { + "type": "date" + } + } + } +} \ No newline at end of file diff --git a/client-adapter/es6x/src/test/resources/es6/mytest_user_single.yml_ b/client-adapter/es6x/src/test/resources/es6/mytest_user_single.yml_ new file mode 100644 index 00000000..23e1b491 --- /dev/null +++ b/client-adapter/es6x/src/test/resources/es6/mytest_user_single.yml_ @@ -0,0 +1,8 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a" + commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/es6x/src/test/resources/log4j2-test.xml b/client-adapter/es6x/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..9c3d2ee6 --- /dev/null +++ b/client-adapter/es6x/src/test/resources/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/client-adapter/es6x/src/test/resources/logback-test.xml b/client-adapter/es6x/src/test/resources/logback-test.xml new file mode 100644 index 00000000..91f0de80 --- /dev/null +++ b/client-adapter/es6x/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/client-adapter/es7x/pom.xml b/client-adapter/es7x/pom.xml new file mode 100644 index 00000000..8083d176 --- /dev/null +++ b/client-adapter/es7x/pom.xml @@ -0,0 +1,100 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.5-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.es7x + jar + canal client adapter es v7x module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.common + ${project.version} + provided + + + com.alibaba.otter + client-adapter.es-core + ${project.version} + + + org.elasticsearch + elasticsearch + 7.3.0 + + + org.elasticsearch.client + transport + 7.3.0 + + + org.elasticsearch.client + elasticsearch-rest-client + 7.3.0 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 7.3.0 + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + maven-antrun-plugin + + + package + + run + + + + + + + + + + + + + + + + diff --git a/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java new file mode 100644 index 00000000..da4b36ba --- /dev/null +++ b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java @@ -0,0 +1,123 @@ +package com.alibaba.otter.canal.client.adapter.es7x; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import org.elasticsearch.action.search.SearchResponse; + +import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate; +import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.SPI; + +import javax.sql.DataSource; + +/** + * ES 7.x 外部适配器 + * + * @author rewerma 2019-09-23 + * @version 1.0.0 + */ +@SPI("es7") +public class ES7xAdapter extends ESAdapter { + + private ESConnection esConnection; + + public ESConnection getEsConnection() { + return esConnection; + } + + @Override + public void init(OuterAdapterConfig configuration, Properties envProperties) { + try { + Map properties = configuration.getProperties(); + + String[] hostArray = configuration.getHosts().split(","); + String mode = properties.get("mode"); + if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) { + esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST); + } else { + esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT); + } + this.esTemplate = new ES7xTemplate(esConnection); + + envProperties.put("es.version", "es7"); + super.init(configuration, envProperties); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public Map count(String task) { + ESSyncConfig config = esSyncConfig.get(task); + ESSyncConfig.ESMapping mapping = config.getEsMapping(); + SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index()).size(0).getResponse(); + + long rowCount = response.getHits().getTotalHits().value; + Map res = new LinkedHashMap<>(); + res.put("esIndex", mapping.get_index()); + res.put("count", rowCount); + return res; + } + + @Override + public EtlResult etl(String task, List params) { + EtlResult etlResult = new EtlResult(); + ESSyncConfig config = esSyncConfig.get(task); + if (config != null) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + ESEtlService esEtlService = new ESEtlService(esConnection, config); + if (dataSource != null) { + return esEtlService.importData(params); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } + } else { + StringBuilder resultMsg = new StringBuilder(); + boolean resSuccess = true; + for (ESSyncConfig configTmp : esSyncConfig.values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + ESEtlService esEtlService = new ESEtlService(esConnection, configTmp); + EtlResult etlRes = esEtlService.importData(params); + if (!etlRes.getSucceeded()) { + resSuccess = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSuccess); + if (resSuccess) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; + } + + @Override + public void destroy() { + super.destroy(); + if (esConnection != null) { + esConnection.close(); + } + } +} diff --git a/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/etl/ESEtlService.java b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/etl/ESEtlService.java new file mode 100644 index 00000000..05b20cf9 --- /dev/null +++ b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/etl/ESEtlService.java @@ -0,0 +1,202 @@ +package com.alibaba.otter.canal.client.adapter.es7x.etl; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate; +import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection; +import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection.ESSearchRequest; +import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.Util; + +/** + * ES ETL Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESEtlService extends AbstractEtlService { + + private ESConnection esConnection; + private ESTemplate esTemplate; + private ESSyncConfig config; + + public ESEtlService(ESConnection esConnection, ESSyncConfig config){ + super("ES", config); + this.esConnection = esConnection; + this.esTemplate = new ES7xTemplate(esConnection); + this.config = config; + } + + public EtlResult importData(List params) { + ESMapping mapping = config.getEsMapping(); + logger.info("start etl to import data to index: {}", mapping.get_index()); + String sql = mapping.getSql(); + return importData(sql, params); + } + + protected boolean executeSqlImport(DataSource ds, String sql, List values, + AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount, + List errMsg) { + try { + ESMapping mapping = (ESMapping) adapterMapping; + Util.sqlRS(ds, sql, values, rs -> { + int count = 0; + try { + ESBulkRequest esBulkRequest = this.esConnection.new ES7xBulkRequest(); + + long batchBegin = System.currentTimeMillis(); + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = null; + for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) { + + String fieldName = fieldItem.getFieldName(); + if (mapping.getSkips().contains(fieldName)) { + continue; + } + + // 如果是主键字段则不插入 + if (fieldItem.getFieldName().equals(mapping.get_id())) { + idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName); + } else { + Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName); + esFieldData.put(Util.cleanColumn(fieldName), val); + } + + } + + if (!mapping.getRelations().isEmpty()) { + mapping.getRelations().forEach((relationField, relationMapping) -> { + Map relations = new HashMap<>(); + relations.put("name", relationMapping.getName()); + if (StringUtils.isNotEmpty(relationMapping.getParent())) { + FieldItem parentFieldItem = mapping.getSchemaItem() + .getSelectFields() + .get(relationMapping.getParent()); + Object parentVal; + try { + parentVal = esTemplate.getValFromRS(mapping, + rs, + parentFieldItem.getFieldName(), + parentFieldItem.getFieldName()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + if (parentVal != null) { + relations.put("parent", parentVal.toString()); + esFieldData.put("$parent_routing", parentVal.toString()); + + } + } + esFieldData.put(Util.cleanColumn(relationField), relations); + }); + } + + if (idVal != null) { + String parentVal = (String) esFieldData.remove("$parent_routing"); + if (mapping.isUpsert()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest( + mapping.get_index(), + idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); + + if (StringUtils.isNotEmpty(parentVal)) { + esUpdateRequest.setRouting(parentVal); + } + + esBulkRequest.add(esUpdateRequest); + } else { + ESIndexRequest esIndexRequest = this.esConnection.new ES7xIndexRequest( + mapping.get_index(), + idVal.toString()).setSource(esFieldData); + if (StringUtils.isNotEmpty(parentVal)) { + esIndexRequest.setRouting(parentVal); + } + esBulkRequest.add(esIndexRequest); + } + } else { + idVal = esFieldData.get(mapping.getPk()); + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)) + .size(10000); + SearchResponse response = esSearchRequest.getResponse(); + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest( + mapping.get_index(), + hit.getId()).setDoc(esFieldData); + esBulkRequest.add(esUpdateRequest); + } + } + + if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 + && esBulkRequest.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + ESBulkResponse rp = esBulkRequest.bulk(); + if (rp.hasFailures()) { + rp.processFailBulkResponse("全量数据 etl 异常 "); + } + + if (logger.isTraceEnabled()) { + logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + esBulkRequest.numberOfActions(), + mapping.get_index()); + } + batchBegin = System.currentTimeMillis(); + esBulkRequest.resetBulk(); + } + count++; + impCount.incrementAndGet(); + } + + if (esBulkRequest.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + ESBulkResponse rp = esBulkRequest.bulk(); + if (rp.hasFailures()) { + rp.processFailBulkResponse("全量数据 etl 异常 "); + } + if (logger.isTraceEnabled()) { + logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + esBulkRequest.numberOfActions(), + mapping.get_index()); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage()); + throw new RuntimeException(e); + } + return count; + }); + + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } +} diff --git a/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java new file mode 100644 index 00000000..537b6bef --- /dev/null +++ b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java @@ -0,0 +1,462 @@ +package com.alibaba.otter.canal.client.adapter.es7x.support; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection.ESSearchRequest; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class ES7xTemplate implements ESTemplate { + + private static final Logger logger = LoggerFactory + .getLogger(ESTemplate.class); + + private static final int MAX_BATCH_SIZE = 1000; + + private ESConnection esConnection; + + private ESBulkRequest esBulkRequest; + + // es 字段类型本地缓存 + private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + + public ES7xTemplate(ESConnection esConnection){ + this.esConnection = esConnection; + this.esBulkRequest = this.esConnection.new ES7xBulkRequest(); + } + + public ESBulkRequest getBulk() { + return esBulkRequest; + } + + public void resetBulkRequestBuilder() { + this.esBulkRequest.resetBulk(); + } + + @Override + public void insert(ESMapping mapping, Object pkVal, Map esFieldData) { + if (mapping.get_id() != null) { + String parentVal = (String) esFieldData.remove("$parent_routing"); + if (mapping.isUpsert()) { + ESUpdateRequest updateRequest = esConnection.new ES7xUpdateRequest(mapping.get_index(), + pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); + if (StringUtils.isNotEmpty(parentVal)) { + updateRequest.setRouting(parentVal); + } + getBulk().add(updateRequest); + } else { + ESIndexRequest indexRequest = esConnection.new ES7xIndexRequest(mapping.get_index(), pkVal.toString()) + .setSource(esFieldData); + if (StringUtils.isNotEmpty(parentVal)) { + indexRequest.setRouting(parentVal); + } + getBulk().add(indexRequest); + } + commitBulk(); + } else { + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .size(10000); + SearchResponse response = esSearchRequest.getResponse(); + + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), + hit.getId()).setDoc(esFieldData); + getBulk().add(esUpdateRequest); + commitBulk(); + } + } + } + + @Override + public void update(ESMapping mapping, Object pkVal, Map esFieldData) { + Map esFieldDataTmp = new LinkedHashMap<>(esFieldData.size()); + esFieldData.forEach((k, v) -> esFieldDataTmp.put(Util.cleanColumn(k), v)); + append4Update(mapping, pkVal, esFieldDataTmp); + commitBulk(); + } + + @Override + public void updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { + if (paramsTmp.isEmpty()) { + return; + } + ESMapping mapping = config.getEsMapping(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); + + // 查询sql批量更新 + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); + List values = new ArrayList<>(); + paramsTmp.forEach((fieldName, value) -> { + sql.append("_v.").append(fieldName).append("=? AND "); + values.add(value); + }); + // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件 + int len = sql.length(); + sql.delete(len - 4, len); + Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> { + int count = 0; + try { + while (rs.next()) { + Object idVal = getIdValFromRS(mapping, rs); + append4Update(mapping, idVal, esFieldData); + commitBulk(); + count++; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return count; + }); + if (logger.isTraceEnabled()) { + logger.trace("Update ES by query affected {} records", syncCount); + } + } + + @Override + public void delete(ESMapping mapping, Object pkVal, Map esFieldData) { + if (mapping.get_id() != null) { + ESDeleteRequest esDeleteRequest = this.esConnection.new ES7xDeleteRequest(mapping.get_index(), + pkVal.toString()); + getBulk().add(esDeleteRequest); + commitBulk(); + } else { + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .size(10000); + SearchResponse response = esSearchRequest.getResponse(); + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), + hit.getId()).setDoc(esFieldData); + getBulk().add(esUpdateRequest); + commitBulk(); + } + } + } + + @Override + public void commit() { + if (getBulk().numberOfActions() > 0) { + ESBulkResponse response = getBulk().bulk(); + if (response.hasFailures()) { + response.processFailBulkResponse("ES sync commit error "); + } + resetBulkRequestBuilder(); + } + } + + @Override + public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException { + fieldName = Util.cleanColumn(fieldName); + columnName = Util.cleanColumn(columnName); + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(columnName); + if (value instanceof Boolean) { + if (!"boolean".equals(esType)) { + value = resultSet.getByte(columnName); + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + @Override + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); + } + } + + // 添加父子文档关联信息 + putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData); + + return resultIdVal; + } + + @Override + public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + break; + } + } + return resultIdVal; + } + + @Override + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + } + + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (dmlOld.containsKey(columnItem.getColumnName()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), + getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); + break; + } + } + } + + // 添加父子文档关联信息 + putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData); + + return resultIdVal; + } + + @Override + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(columnName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + @Override + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value); + } + } + + // 添加父子文档关联信息 + putRelationData(mapping, schemaItem, dmlData, esFieldData); + return resultIdVal; + } + + @Override + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + } + + if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), + getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); + } + } + + // 添加父子文档关联信息 + putRelationData(mapping, schemaItem, dmlOld, esFieldData); + return resultIdVal; + } + + /** + * 如果大于批量数则提交批次 + */ + private void commitBulk() { + if (getBulk().numberOfActions() >= MAX_BATCH_SIZE) { + commit(); + } + } + + private void append4Update(ESMapping mapping, Object pkVal, Map esFieldData) { + if (mapping.get_id() != null) { + String parentVal = (String) esFieldData.remove("$parent_routing"); + if (mapping.isUpsert()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), + pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); + if (StringUtils.isNotEmpty(parentVal)) { + esUpdateRequest.setRouting(parentVal); + } + getBulk().add(esUpdateRequest); + } else { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), + pkVal.toString()).setDoc(esFieldData); + if (StringUtils.isNotEmpty(parentVal)) { + esUpdateRequest.setRouting(parentVal); + } + getBulk().add(esUpdateRequest); + } + } else { + ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .size(10000); + SearchResponse response = esSearchRequest.getResponse(); + for (SearchHit hit : response.getHits()) { + ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), + hit.getId()).setDoc(esFieldData); + getBulk().add(esUpdateRequest); + } + } + } + + /** + * 获取es mapping中的属性类型 + * + * @param mapping mapping配置 + * @param fieldName 属性名 + * @return 类型 + */ + @SuppressWarnings("unchecked") + private String getEsType(ESMapping mapping, String fieldName) { + String key = mapping.get_index() + "-" + mapping.get_type(); + Map fieldType = esFieldTypes.get(key); + if (fieldType != null) { + return fieldType.get(fieldName); + } else { + MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index()); + + if (mappingMetaData == null) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + + fieldType = new LinkedHashMap<>(); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); + if (value.containsKey("properties")) { + fieldType.put(entry.getKey(), "object"); + } else { + fieldType.put(entry.getKey(), (String) value.get("type")); + } + } + esFieldTypes.put(key, fieldType); + + return fieldType.get(fieldName); + } + } + + private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet, + Map esFieldData) { + // 添加父子文档关联信息 + if (!mapping.getRelations().isEmpty()) { + mapping.getRelations().forEach((relationField, relationMapping) -> { + Map relations = new HashMap<>(); + relations.put("name", relationMapping.getName()); + if (StringUtils.isNotEmpty(relationMapping.getParent())) { + FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent()); + Object parentVal; + try { + parentVal = getValFromRS(mapping, + resultSet, + parentFieldItem.getFieldName(), + parentFieldItem.getFieldName()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + if (parentVal != null) { + relations.put("parent", parentVal.toString()); + esFieldData.put("$parent_routing", parentVal.toString()); + + } + } + esFieldData.put(relationField, relations); + }); + } + } + + private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map dmlData, + Map esFieldData) { + // 添加父子文档关联信息 + if (!mapping.getRelations().isEmpty()) { + mapping.getRelations().forEach((relationField, relationMapping) -> { + Map relations = new HashMap<>(); + relations.put("name", relationMapping.getName()); + if (StringUtils.isNotEmpty(relationMapping.getParent())) { + FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent()); + String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName(); + Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName); + if (parentVal != null) { + relations.put("parent", parentVal.toString()); + esFieldData.put("$parent_routing", parentVal.toString()); + + } + } + esFieldData.put(relationField, relations); + }); + } + } +} diff --git a/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java new file mode 100644 index 00000000..c526721f --- /dev/null +++ b/client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java @@ -0,0 +1,504 @@ +package com.alibaba.otter.canal.client.adapter.es7x.support; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; + +/** + * ES 连接器, Transport Rest 两种方式 + * + * @author rewerma 2019-08-01 + * @version 1.0.0 + */ +public class ESConnection { + + private static final Logger logger = LoggerFactory.getLogger(ESConnection.class); + + public enum ESClientMode { + TRANSPORT, REST + } + + private ESClientMode mode; + + private TransportClient transportClient; + + private RestHighLevelClient restHighLevelClient; + + public ESConnection(String[] hosts, Map properties, ESClientMode mode) throws UnknownHostException{ + this.mode = mode; + if (mode == ESClientMode.TRANSPORT) { + Settings.Builder settingBuilder = Settings.builder(); + settingBuilder.put("cluster.name", properties.get("cluster.name")); + Settings settings = settingBuilder.build(); + transportClient = new PreBuiltTransportClient(settings); + for (String host : hosts) { + int i = host.indexOf(":"); + transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)), + Integer.parseInt(host.substring(i + 1)))); + } + } else { + HttpHost[] httpHosts = new HttpHost[hosts.length]; + for (int i = 0; i < hosts.length; i++) { + String host = hosts[i]; + int j = host.indexOf(":"); + HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)), + Integer.parseInt(host.substring(j + 1))); + httpHosts[i] = httpHost; + } + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + String nameAndPwd = properties.get("security.auth"); + if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) { + String[] nameAndPwdArr = nameAndPwd.split(":"); + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1])); + restClientBuilder.setHttpClientConfigCallback( + httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + restHighLevelClient = new RestHighLevelClient(restClientBuilder); + } + } + + public void close() { + if (mode == ESClientMode.TRANSPORT) { + transportClient.close(); + } else { + try { + restHighLevelClient.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public MappingMetaData getMapping(String index) { + MappingMetaData mappingMetaData = null; + if (mode == ESClientMode.TRANSPORT) { + try { + mappingMetaData = transportClient.admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices() + .get(index) + .mapping(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + index); + } + } else { + Map mappings; + try { + GetMappingsRequest request = new GetMappingsRequest(); + request.indices(index); + GetMappingsResponse response = restHighLevelClient.indices() + .getMapping(request, RequestOptions.DEFAULT); + + mappings = response.mappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + index); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return null; + } + mappingMetaData = mappings.get(index); + } + return mappingMetaData; + } + + public class ES7xIndexRequest implements ESBulkRequest.ESIndexRequest { + + private IndexRequestBuilder indexRequestBuilder; + + private IndexRequest indexRequest; + + public ES7xIndexRequest(String index, String id){ + if (mode == ESClientMode.TRANSPORT) { + indexRequestBuilder = transportClient.prepareIndex(); + indexRequestBuilder.setIndex(index); + indexRequestBuilder.setId(id); + } else { + indexRequest = new IndexRequest(index); + indexRequest.id(id); + } + } + + public ES7xIndexRequest setSource(Map source) { + if (mode == ESClientMode.TRANSPORT) { + indexRequestBuilder.setSource(source); + } else { + indexRequest.source(source); + } + return this; + } + + public ES7xIndexRequest setRouting(String routing) { + if (mode == ESClientMode.TRANSPORT) { + indexRequestBuilder.setRouting(routing); + } else { + indexRequest.routing(routing); + } + return this; + } + + public IndexRequestBuilder getIndexRequestBuilder() { + return indexRequestBuilder; + } + + public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) { + this.indexRequestBuilder = indexRequestBuilder; + } + + public IndexRequest getIndexRequest() { + return indexRequest; + } + + public void setIndexRequest(IndexRequest indexRequest) { + this.indexRequest = indexRequest; + } + } + + public class ES7xUpdateRequest implements ESBulkRequest.ESUpdateRequest { + + private UpdateRequestBuilder updateRequestBuilder; + + private UpdateRequest updateRequest; + + public ES7xUpdateRequest(String index, String id){ + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder = transportClient.prepareUpdate(); + updateRequestBuilder.setIndex(index); + updateRequestBuilder.setId(id); + } else { + updateRequest = new UpdateRequest(index, id); + } + } + + public ES7xUpdateRequest setDoc(Map source) { + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder.setDoc(source); + } else { + updateRequest.doc(source); + } + return this; + } + + public ES7xUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) { + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder.setDocAsUpsert(shouldUpsertDoc); + } else { + updateRequest.docAsUpsert(shouldUpsertDoc); + } + return this; + } + + public ES7xUpdateRequest setRouting(String routing) { + if (mode == ESClientMode.TRANSPORT) { + updateRequestBuilder.setRouting(routing); + } else { + updateRequest.routing(routing); + } + return this; + } + + public UpdateRequestBuilder getUpdateRequestBuilder() { + return updateRequestBuilder; + } + + public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) { + this.updateRequestBuilder = updateRequestBuilder; + } + + public UpdateRequest getUpdateRequest() { + return updateRequest; + } + + public void setUpdateRequest(UpdateRequest updateRequest) { + this.updateRequest = updateRequest; + } + } + + public class ES7xDeleteRequest implements ESBulkRequest.ESDeleteRequest { + + private DeleteRequestBuilder deleteRequestBuilder; + + private DeleteRequest deleteRequest; + + public ES7xDeleteRequest(String index, String id){ + if (mode == ESClientMode.TRANSPORT) { + deleteRequestBuilder = transportClient.prepareDelete(); + deleteRequestBuilder.setIndex(index); + deleteRequestBuilder.setId(id); + } else { + deleteRequest = new DeleteRequest(index, id); + } + } + + public DeleteRequestBuilder getDeleteRequestBuilder() { + return deleteRequestBuilder; + } + + public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) { + this.deleteRequestBuilder = deleteRequestBuilder; + } + + public DeleteRequest getDeleteRequest() { + return deleteRequest; + } + + public void setDeleteRequest(DeleteRequest deleteRequest) { + this.deleteRequest = deleteRequest; + } + } + + public class ESSearchRequest { + + private SearchRequestBuilder searchRequestBuilder; + + private SearchRequest searchRequest; + + private SearchSourceBuilder sourceBuilder; + + public ESSearchRequest(String index){ + if (mode == ESClientMode.TRANSPORT) { + searchRequestBuilder = transportClient.prepareSearch(index); + } else { + searchRequest = new SearchRequest(index); + sourceBuilder = new SearchSourceBuilder(); + } + } + + public ESSearchRequest setQuery(QueryBuilder queryBuilder) { + if (mode == ESClientMode.TRANSPORT) { + searchRequestBuilder.setQuery(queryBuilder); + } else { + sourceBuilder.query(queryBuilder); + } + return this; + } + + public ESSearchRequest size(int size) { + if (mode == ESClientMode.TRANSPORT) { + searchRequestBuilder.setSize(size); + } else { + sourceBuilder.size(size); + } + return this; + } + + public SearchResponse getResponse() { + if (mode == ESClientMode.TRANSPORT) { + return searchRequestBuilder.get(); + } else { + searchRequest.source(sourceBuilder); + try { + return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public SearchRequestBuilder getSearchRequestBuilder() { + return searchRequestBuilder; + } + + public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) { + this.searchRequestBuilder = searchRequestBuilder; + } + + public SearchRequest getSearchRequest() { + return searchRequest; + } + + public void setSearchRequest(SearchRequest searchRequest) { + this.searchRequest = searchRequest; + } + } + + public class ES7xBulkRequest implements ESBulkRequest { + + private BulkRequestBuilder bulkRequestBuilder; + + private BulkRequest bulkRequest; + + public ES7xBulkRequest(){ + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder = transportClient.prepareBulk(); + } else { + bulkRequest = new BulkRequest(); + } + } + + public void resetBulk() { + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder = transportClient.prepareBulk(); + } else { + bulkRequest = new BulkRequest(); + } + } + + public ES7xBulkRequest add(ESIndexRequest esIndexRequest) { + ES7xIndexRequest eir = (ES7xIndexRequest) esIndexRequest; + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder.add(eir.indexRequestBuilder); + } else { + bulkRequest.add(eir.indexRequest); + } + return this; + } + + public ES7xBulkRequest add(ESUpdateRequest esUpdateRequest) { + ES7xUpdateRequest eur = (ES7xUpdateRequest) esUpdateRequest; + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder.add(eur.updateRequestBuilder); + } else { + bulkRequest.add(eur.updateRequest); + } + return this; + } + + public ES7xBulkRequest add(ESDeleteRequest esDeleteRequest) { + ES7xDeleteRequest edr = (ES7xDeleteRequest) esDeleteRequest; + if (mode == ESClientMode.TRANSPORT) { + bulkRequestBuilder.add(edr.deleteRequestBuilder); + } else { + bulkRequest.add(edr.deleteRequest); + } + return this; + } + + public int numberOfActions() { + if (mode == ESClientMode.TRANSPORT) { + return bulkRequestBuilder.numberOfActions(); + } else { + return bulkRequest.numberOfActions(); + } + } + + public ESBulkResponse bulk() { + if (mode == ESClientMode.TRANSPORT) { + BulkResponse responses = bulkRequestBuilder.execute().actionGet(); + return new ES7xBulkResponse(responses); + } else { + try { + BulkResponse responses = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT); + return new ES7xBulkResponse(responses); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public BulkRequestBuilder getBulkRequestBuilder() { + return bulkRequestBuilder; + } + + public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) { + this.bulkRequestBuilder = bulkRequestBuilder; + } + + public BulkRequest getBulkRequest() { + return bulkRequest; + } + + public void setBulkRequest(BulkRequest bulkRequest) { + this.bulkRequest = bulkRequest; + } + } + + public static class ES7xBulkResponse implements ESBulkRequest.ESBulkResponse { + + private BulkResponse bulkResponse; + + public ES7xBulkResponse(BulkResponse bulkResponse){ + this.bulkResponse = bulkResponse; + } + + @Override + public boolean hasFailures() { + return bulkResponse.hasFailures(); + } + + @Override + public void processFailBulkResponse(String errorMsg) { + for (BulkItemResponse itemResponse : bulkResponse.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.error(itemResponse.getFailureMessage()); + } else { + throw new RuntimeException(errorMsg + itemResponse.getFailureMessage()); + } + } + } + } + + // ------ get/set ------ + public ESClientMode getMode() { + return mode; + } + + public void setMode(ESClientMode mode) { + this.mode = mode; + } + + public TransportClient getTransportClient() { + return transportClient; + } + + public void setTransportClient(TransportClient transportClient) { + this.transportClient = transportClient; + } + + public RestHighLevelClient getRestHighLevelClient() { + return restHighLevelClient; + } + + public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + } +} diff --git a/client-adapter/es7x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter b/client-adapter/es7x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter new file mode 100644 index 00000000..d28f8641 --- /dev/null +++ b/client-adapter/es7x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter @@ -0,0 +1 @@ +es7=com.alibaba.otter.canal.client.adapter.es7x.ES7xAdapter diff --git a/client-adapter/es7x/src/main/resources/es7/biz_order.yml b/client-adapter/es7x/src/main/resources/es7/biz_order.yml new file mode 100644 index 00000000..86a22583 --- /dev/null +++ b/client-adapter/es7x/src/main/resources/es7/biz_order.yml @@ -0,0 +1,20 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: customer + _id: _id + relations: + customer_order: + name: order + parent: customer_id + sql: "select concat('oid_', t.id) as _id, + t.customer_id, + t.id as order_id, + t.serial_code as order_serial, + t.c_time as order_time + from biz_order t" + skips: + - customer_id + etlCondition: "where t.c_time>={}" + commitBatch: 3000 diff --git a/client-adapter/es7x/src/main/resources/es7/customer.yml b/client-adapter/es7x/src/main/resources/es7/customer.yml new file mode 100644 index 00000000..4010dcff --- /dev/null +++ b/client-adapter/es7x/src/main/resources/es7/customer.yml @@ -0,0 +1,46 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: customer + _id: id + relations: + customer_order: + name: customer + sql: "select t.id, t.name, t.email from customer t" + etlCondition: "where t.c_time>={}" + commitBatch: 3000 + + +#{ +# "mappings":{ +# "_doc":{ +# "properties":{ +# "id": { +# "type": "long" +# }, +# "name": { +# "type": "text" +# }, +# "email": { +# "type": "text" +# }, +# "order_id": { +# "type": "long" +# }, +# "order_serial": { +# "type": "text" +# }, +# "order_time": { +# "type": "date" +# }, +# "customer_order":{ +# "type":"join", +# "relations":{ +# "customer":"order" +# } +# } +# } +# } +# } +#} diff --git a/client-adapter/es7x/src/main/resources/es7/mytest_user.yml b/client-adapter/es7x/src/main/resources/es7/mytest_user.yml new file mode 100644 index 00000000..9f9c5b8d --- /dev/null +++ b/client-adapter/es7x/src/main/resources/es7/mytest_user.yml @@ -0,0 +1,15 @@ +dataSourceKey: defaultDS +destination: example +groupId: g1 +esMapping: + _index: mytest_user + _id: _id +# upsert: true +# pk: id + sql: "select a.id as _id, a.name, a.role_id, b.role_name, + a.c_time from user a + left join role b on b.id=a.role_id" +# objFields: +# _labels: array:; + etlCondition: "where a.c_time>={}" + commitBatch: 3000 diff --git a/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java b/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java new file mode 100644 index 00000000..6aa6ade8 --- /dev/null +++ b/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java @@ -0,0 +1,119 @@ +package com.alibaba.otter.canal.client.adapter.es7x.test; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class ES7xTest { + + private TransportClient transportClient; + + @Before + public void init() throws UnknownHostException { + Settings.Builder settingBuilder = Settings.builder(); + settingBuilder.put("cluster.name", TestConstant.clusterName); + Settings settings = settingBuilder.build(); + transportClient = new PreBuiltTransportClient(settings); + String[] hostArray = TestConstant.esHosts.split(","); + for (String host : hostArray) { + int i = host.indexOf(":"); + transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)), + Integer.parseInt(host.substring(i + 1)))); + } + } + + @Test + public void test01() { + SearchResponse response = transportClient.prepareSearch("test") + .setQuery(QueryBuilders.termQuery("_id", "1")) + .setSize(10000) + .get(); + for (SearchHit hit : response.getHits()) { + System.out.println(hit.getSourceAsMap().get("data").getClass()); + } + } + + @Test + public void test02() { + Map esFieldData = new LinkedHashMap<>(); + esFieldData.put("userId", 2L); + esFieldData.put("eventId", 4L); + esFieldData.put("eventName", "网络异常"); + esFieldData.put("description", "第四个事件信息"); + + Map relations = new LinkedHashMap<>(); + esFieldData.put("user_event", relations); + relations.put("name", "event"); + relations.put("parent", "2"); + + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + bulkRequestBuilder + .add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData)); + commit(bulkRequestBuilder); + } + + @Test + public void test03() { + Map esFieldData = new LinkedHashMap<>(); + esFieldData.put("userId", 2L); + esFieldData.put("eventName", "网络异常1"); + + Map relations = new LinkedHashMap<>(); + esFieldData.put("user_event", relations); + relations.put("name", "event"); + relations.put("parent", "2"); + + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + bulkRequestBuilder.add(transportClient.prepareUpdate("test", "osm", "2_4").setRouting("2").setDoc(esFieldData)); + commit(bulkRequestBuilder); + } + + @Test + public void test04() { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + bulkRequestBuilder.add(transportClient.prepareDelete("test", "osm", "2_4")); + commit(bulkRequestBuilder); + } + + private void commit(BulkRequestBuilder bulkRequestBuilder) { + if (bulkRequestBuilder.numberOfActions() > 0) { + BulkResponse response = bulkRequestBuilder.execute().actionGet(); + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + System.out.println(itemResponse.getFailureMessage()); + } else { + System.out.println("ES bulk commit error" + itemResponse.getFailureMessage()); + } + } + } + } + } + + @After + public void after() { + transportClient.close(); + } +} diff --git a/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java b/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java new file mode 100644 index 00000000..7a36db23 --- /dev/null +++ b/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java @@ -0,0 +1,45 @@ +package com.alibaba.otter.canal.client.adapter.es7x.test; + +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; + +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection; +import org.springframework.util.Assert; + +@Ignore +public class ESConnectionTest { + + ESConnection esConnection; + + @Before + public void init() throws UnknownHostException { + String[] hosts = new String[] { "127.0.0.1:9200" }; + Map properties = new HashMap<>(); + properties.put("cluster.name", "elasticsearch"); + esConnection = new ESConnection(hosts, properties, ESConnection.ESClientMode.REST); + } + + @Test + public void test01() { + MappingMetaData mappingMetaData = esConnection.getMapping("mytest_user"); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); + if (value.containsKey("properties")) { + System.out.println(entry.getKey() + " object"); + } else { + System.out.println(entry.getKey() + " " + value.get("type")); + Assert.notNull(entry.getKey(), "null column name"); + Assert.notNull(value.get("type"), "null column type"); + } + } + } +} diff --git a/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/TestConstant.java b/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/TestConstant.java new file mode 100644 index 00000000..273bb7f5 --- /dev/null +++ b/client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/TestConstant.java @@ -0,0 +1,40 @@ +package com.alibaba.otter.canal.client.adapter.es7x.test; + +import java.sql.SQLException; + +import com.alibaba.druid.pool.DruidDataSource; + +public class TestConstant { + + public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"; + public final static String jdbcUser = "root"; + public final static String jdbcPassword = "121212"; + + public final static String esHosts = "127.0.0.1:9300"; + public final static String clusterName = "elasticsearch"; + + public final static DruidDataSource dataSource; + + static { + dataSource = new DruidDataSource(); + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUser); + dataSource.setPassword(jdbcPassword); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(1); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setPoolPreparedStatements(false); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + dataSource.setValidationQuery("select 1"); + try { + dataSource.init(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 879a0cbc..e6abb92c 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -132,7 +132,20 @@ com.alibaba.otter - client-adapter.elasticsearch + client-adapter.es6x + ${project.version} + + + * + * + + + jar-with-dependencies + provided + + + com.alibaba.otter + client-adapter.es7x ${project.version} diff --git a/client-adapter/launcher/src/main/assembly/dev.xml b/client-adapter/launcher/src/main/assembly/dev.xml index 7beaf269..bf4c5afa 100644 --- a/client-adapter/launcher/src/main/assembly/dev.xml +++ b/client-adapter/launcher/src/main/assembly/dev.xml @@ -30,8 +30,15 @@ - ../elasticsearch/src/main/resources/es - /conf/es + ../es6x/src/main/resources/es6 + /conf/es6 + + **/* + + + + ../es7x/src/main/resources/es7 + /conf/es7 **/* diff --git a/client-adapter/launcher/src/main/assembly/release.xml b/client-adapter/launcher/src/main/assembly/release.xml index 456b4a6a..1d2367ff 100644 --- a/client-adapter/launcher/src/main/assembly/release.xml +++ b/client-adapter/launcher/src/main/assembly/release.xml @@ -31,8 +31,15 @@ - ../elasticsearch/src/main/resources/es - /conf/es + ../es6x/src/main/resources/es6 + /conf/es6 + + **/* + + + + ../es7x/src/main/resources/es7 + /conf/es7 **/* @@ -71,4 +78,4 @@ - \ No newline at end of file + diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index 7990e6eb..8d261256 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -23,6 +23,9 @@ elasticsearch launcher rdb + es_6.x + es_7.x + es-core