From 0f14ed926bc44f74f38e3e0b54d7dfa2d4c860ca Mon Sep 17 00:00:00 2001 From: wuwo Date: Mon, 29 Oct 2018 17:32:10 +0800 Subject: [PATCH 01/42] pass useDruidDdlFilter parameter to LogEventConvert --- .../canal/parse/inbound/mysql/AbstractMysqlEventParser.java | 1 + .../canal/parse/inbound/mysql/dbsync/LogEventConvert.java | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 8050186d..5e94e40b 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -60,6 +60,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { convert.setFilterQueryDdl(filterQueryDdl); convert.setFilterRows(filterRows); convert.setFilterTableError(filterTableError); + convert.setUseDruidDdlFilter(useDruidDdlFilter); return convert; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index ec71465b..5a302b86 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -981,4 +981,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar public void setFilterRows(boolean filterRows) { this.filterRows = filterRows; } + + public void setUseDruidDdlFilter(boolean useDruidDdlFilter) { + this.useDruidDdlFilter = useDruidDdlFilter; + } } From 4c7129115c56fca0949b4e046375b5cd498d5bcd Mon Sep 17 00:00:00 2001 From: wuwo Date: Mon, 29 Oct 2018 17:49:34 +0800 Subject: [PATCH 02/42] a little improve about tsdb --- .../inbound/mysql/dbsync/LogEventConvert.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 5a302b86..5288ac2a 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -233,7 +233,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar if (useDruidDdlFilter) { List results = DruidDdlParser.parse(queryString, event.getDbName()); for (DdlResult result : results) { - if (!processFilter(queryString, result)) { + if (!processFilter(queryString, result, event, isSeek)) { // 只要有一个数据不进行过滤 notFilter = true; } @@ -246,7 +246,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } else { DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName()); - if (!processFilter(queryString, result)) { + if (!processFilter(queryString, result, event, isSeek)) { notFilter = true; } @@ -260,12 +260,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return null; } - if (!isSeek) { - // 使用新的表结构元数据管理方式 - EntryPosition position = createPosition(event.getHeader()); - tableMetaCache.apply(position, event.getDbName(), queryString, null); - } - Header header = createHeader(event.getHeader(), schemaName, tableName, type); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); if (type != EventType.QUERY && type != EventType.INSERT && type != EventType.UPDATE @@ -285,7 +279,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return StringUtils.substringAfter(queryString, type); } - private boolean processFilter(String queryString, DdlResult result) { + private boolean processFilter(String queryString, DdlResult result, QueryLogEvent event, boolean isSeek) { String schemaName = result.getSchemaName(); String tableName = result.getTableName(); // fixed issue https://github.com/alibaba/canal/issues/58 @@ -312,10 +306,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) { // 针对DDL类型 - if (filterQueryDdl) { - return true; - } - if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) { // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析 @@ -351,6 +341,16 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } } + + if (!isSeek) { + // 使用新的表结构元数据管理方式 + EntryPosition position = createPosition(event.getHeader()); + tableMetaCache.apply(position, event.getDbName(), queryString, null); + } + + if (filterQueryDdl) { + return true; + } } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) { // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤 From 4e18d7800ac72857d52b1b5631bc5c41d0f4e6f4 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 09:08:34 +0800 Subject: [PATCH 03/42] =?UTF-8?q?=E6=8B=BC=E5=86=99=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/client/adapter/hbase/HbaseAdapter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index e97035ed..52b3da23 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -62,10 +62,10 @@ public class HbaseAdapter implements OuterAdapter { } } - Map propertites = configuration.getProperties(); + Map properties = configuration.getProperties(); Configuration hbaseConfig = HBaseConfiguration.create(); - propertites.forEach(hbaseConfig::set); + properties.forEach(hbaseConfig::set); conn = ConnectionFactory.createConnection(hbaseConfig); hbaseTemplate = new HbaseTemplate(conn); hbaseSyncService = new HbaseSyncService(hbaseTemplate); From 82b8b56aa3f969f495ca53a954e83ac55f896ec0 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 09:10:49 +0800 Subject: [PATCH 04/42] =?UTF-8?q?partition=E9=BB=98=E8=AE=A4=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deployer/src/main/resources/mq.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployer/src/main/resources/mq.yml b/deployer/src/main/resources/mq.yml index 019d2123..1797bd9b 100644 --- a/deployer/src/main/resources/mq.yml +++ b/deployer/src/main/resources/mq.yml @@ -1,4 +1,4 @@ -servers: localhost:9876 #for rocketmq: means the nameserver +servers: slave1:6667 #for rocketmq: means the nameserver retries: 0 batchSize: 16384 lingerMs: 1 @@ -13,7 +13,7 @@ flatMessage: true canalDestinations: - canalDestination: example topic: example - partition: 1 + partition: # #对应topic分区数量 # partitionsNum: 3 # partitionHash: From 1d2e85672ca85dda301ab04c02d6a93654d7f5c0 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 09:49:04 +0800 Subject: [PATCH 05/42] =?UTF-8?q?elasticsearch=E9=80=82=E9=85=8D=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/elasticsearch/pom.xml | 77 ++++ .../canal/client/adapter/es/ESAdapter.java | 82 ++++ .../adapter/es/config/ESSyncConfig.java | 137 +++++++ .../adapter/es/config/ESSyncConfigLoader.java | 89 +++++ .../client/adapter/es/config/SchemaItem.java | 364 ++++++++++++++++++ .../client/adapter/es/config/SqlParser.java | 205 ++++++++++ .../adapter/es/service/ESSyncService.java | 15 + ...ba.otter.canal.client.adapter.OuterAdapter | 1 + .../src/main/resources/es/myetst_user.yml | 12 + .../adapter/es/test/ConfigLoadTest.java | 31 ++ .../client/adapter/es/test/SqlParseTest.java | 51 +++ client-adapter/pom.xml | 1 + 12 files changed, 1065 insertions(+) create mode 100644 client-adapter/elasticsearch/pom.xml create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java create mode 100644 client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter create mode 100644 client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java diff --git a/client-adapter/elasticsearch/pom.xml b/client-adapter/elasticsearch/pom.xml new file mode 100644 index 00000000..b5a8c11b --- /dev/null +++ b/client-adapter/elasticsearch/pom.xml @@ -0,0 +1,77 @@ + + + + canal.client-adapter + com.alibaba.otter + 1.1.1-SNAPSHOT + + 4.0.0 + com.alibaba.otter + client-adapter.elasticsearch + jar + canal client adapter ElasticSearch module for otter ${project.version} + + + + com.alibaba.otter + client-adapter.common + ${project.version} + provided + + + org.yaml + snakeyaml + 1.19 + provided + + + com.alibaba.fastsql + fastsql + 2.0.0_preview_644 + + + org.elasticsearch + elasticsearch + 6.2.3 + + + org.elasticsearch.client + transport + 6.2.3 + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + + \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java new file mode 100644 index 00000000..8e2523e4 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -0,0 +1,82 @@ +package com.alibaba.otter.canal.client.adapter.es; + +import java.util.List; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; +import com.alibaba.otter.canal.client.adapter.support.SPI; + +/** + * ES外部适配器 + * + * @author rewerma 2018-10-20 + * @version 1.0.0 + */ +@SPI("es") +public class ESAdapter implements OuterAdapter { + + private static Logger logger = LoggerFactory.getLogger(ESAdapter.class); + + private static volatile Map esSyncConfig = null; // 文件名对应配置 + + private TransportClient transportClient; + + private ESSyncService esSyncService; + + @Override + public void init(OuterAdapterConfig configuration) { + try { + if (esSyncConfig == null) { + synchronized (ESSyncConfig.class) { + if (esSyncConfig == null) { + esSyncConfig = ESSyncConfigLoader.load(); + } + } + } + Map properties = configuration.getProperties(); + Settings.Builder settingBuilder = Settings.builder(); + properties.forEach(settingBuilder::put); + Settings settings = settingBuilder.build(); + transportClient = new PreBuiltTransportClient(settings); + esSyncService = new ESSyncService(transportClient); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void sync(Dml dml) { + } + + @Override + public EtlResult etl(String task, List params) { + return null; + } + + @Override + public Map count(String task) { + return null; + } + + @Override + public void destroy() { + + } + + @Override + public String getDestination(String task) { + return null; + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java new file mode 100644 index 00000000..be0b712b --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -0,0 +1,137 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.util.ArrayList; +import java.util.List; + +public class ESSyncConfig { + + private String dataSourceKey; // 数据源key + + private ESMapping esMapping; + + public void validate() { + if (esMapping._index == null) { + throw new NullPointerException("esMapping._index"); + } + if (esMapping._type == null) { + throw new NullPointerException("esMapping._type"); + } + if (esMapping._id == null && esMapping.pk == null) { + throw new NullPointerException("esMapping._id and esMapping.pk"); + } + if (esMapping.sql == null) { + throw new NullPointerException("esMapping.sql"); + } + } + + public String getDataSourceKey() { + return dataSourceKey; + } + + public void setDataSourceKey(String dataSourceKey) { + this.dataSourceKey = dataSourceKey; + } + + public ESMapping getEsMapping() { + return esMapping; + } + + public void setEsMapping(ESMapping esMapping) { + this.esMapping = esMapping; + } + + public static class ESMapping { + + private String _index; + private String _type; + private String _id; + private String pk; + private String parent; + private String sql; + private List skips = new ArrayList<>(); + private boolean alwaysSql = false; + private int commitBatch = 1000; + private String etlCondition; + + public String get_index() { + return _index; + } + + public void set_index(String _index) { + this._index = _index; + } + + public String get_type() { + return _type; + } + + public void set_type(String _type) { + this._type = _type; + } + + public String get_id() { + return _id; + } + + public void set_id(String _id) { + this._id = _id; + } + + public String getPk() { + return pk; + } + + public void setPk(String pk) { + this.pk = pk; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + + public List getSkips() { + return skips; + } + + public void setSkips(List skips) { + this.skips = skips; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public boolean isAlwaysSql() { + return alwaysSql; + } + + public void setAlwaysSql(boolean alwaysSql) { + this.alwaysSql = alwaysSql; + } + + public int getCommitBatch() { + return commitBatch; + } + + public void setCommitBatch(int commitBatch) { + this.commitBatch = commitBatch; + } + + public String getEtlCondition() { + return etlCondition; + } + + public void setEtlCondition(String etlCondition) { + this.etlCondition = etlCondition; + } + + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java new file mode 100644 index 00000000..7e156409 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -0,0 +1,89 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; + +public class ESSyncConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); + + private static final String BASE_PATH = "es"; + + public static Map load() { + logger.info("## Start loading mapping config ... "); + + Map result = new LinkedHashMap<>(); + + Collection configs = AdapterConfigs.get("es"); + for (String c : configs) { + if (c == null) { + continue; + } + c = c.trim(); + if (c.equals("") || c.startsWith("#")) { + continue; + } + + ESSyncConfig config; + String configContent = null; + + if (c.endsWith(".yml")) { + configContent = readConfigContent(BASE_PATH + "/" + c); + } + + config = new Yaml().loadAs(configContent, ESSyncConfig.class); + + try { + config.validate(); + } catch (Exception e) { + throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e); + } + result.put(c, config); + } + + logger.info("## Mapping config loaded"); + return result; + } + + public static String readConfigContent(String config) { + InputStream in = null; + try { + // 先取本地文件，再取类路径 + File configFile = new File("config/" + config); + if (configFile.exists()) { + in = new FileInputStream(configFile); + } else { + in = ESSyncConfigLoader.class.getClassLoader().getResourceAsStream(config); + } + if (in == null) { + throw new RuntimeException("Config file not found."); + } + + byte[] bytes = new byte[in.available()]; + in.read(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Read yml config error ", e); + } finally { + try { + if (in != null) { + in.close(); + } + } catch (IOException e) { + // ignore + } + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java new file mode 100644 index 00000000..88e29eb6 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -0,0 +1,364 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class SchemaItem { + + private Map aliasTableItems = new LinkedHashMap<>(); + private Map selectFields = new LinkedHashMap<>(); + private String sql; + + private volatile Map> tableItemAliases; + private volatile Map> columnFields; + + public void init() { + this.getTableItemAliases(); + this.getColumnFields(); + aliasTableItems.values().forEach(tableItem -> { + tableItem.getRelationTableFields(); + tableItem.getRelationSelectFields(); + }); + } + + public Map getAliasTableItems() { + return aliasTableItems; + } + + public void setAliasTableItems(Map aliasTableItems) { + this.aliasTableItems = aliasTableItems; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Map getSelectFields() { + return selectFields; + } + + public void setSelectFields(Map selectFields) { + this.selectFields = selectFields; + } + + public Map> getTableItemAliases() { + if (tableItemAliases == null) { + synchronized (SchemaItem.class) { + if (tableItemAliases == null) { + tableItemAliases = new LinkedHashMap<>(); + aliasTableItems.forEach((alias, tableItem) -> { + List aliases = tableItemAliases + .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>()); + aliases.add(tableItem); + }); + } + } + } + return tableItemAliases; + } + + public Map> getColumnFields() { + if (columnFields == null) { + synchronized (SchemaItem.class) { + if (columnFields == null) { + columnFields = new LinkedHashMap<>(); + getSelectFields() + .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { + TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); + if (!tableItem.isSubQuery()) { + List fieldItems = columnFields.computeIfAbsent( + columnItem.getOwner() + "." + columnItem.getColumnName(), + k -> new ArrayList<>()); + fieldItems.add(fieldItem); + } else { + tableItem.getSubQueryFields().forEach(subQueryField -> { + List fieldItems = columnFields.computeIfAbsent( + columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), + k -> new ArrayList<>()); + fieldItems.add(fieldItem); + }); + } + })); + } + } + } + return columnFields; + + } + + public TableItem getMainTable() { + if (!aliasTableItems.isEmpty()) { + return aliasTableItems.values().iterator().next(); + } else { + return null; + } + } + + public static class TableItem { + + private SchemaItem schemaItem; + + private String schema; + private String tableName; + private String alias; + private String subQuerySql; + private List subQueryFields = new ArrayList<>(); + private List relationFields = new ArrayList<>(); + + private boolean main; + private boolean subQuery; + + private volatile List relationTableFields; // 当前表关联条件字段 + private volatile List relationSelectFieldItem; // 关联条件字段在select中的对应字段 + + public TableItem(SchemaItem schemaItem){ + this.schemaItem = schemaItem; + } + + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getSubQuerySql() { + return subQuerySql; + } + + public void setSubQuerySql(String subQuerySql) { + this.subQuerySql = subQuerySql; + } + + public boolean isMain() { + return main; + } + + public void setMain(boolean main) { + this.main = main; + } + + public boolean isSubQuery() { + return subQuery; + } + + public void setSubQuery(boolean subQuery) { + this.subQuery = subQuery; + } + + public List getSubQueryFields() { + return subQueryFields; + } + + public void setSubQueryFields(List subQueryFields) { + this.subQueryFields = subQueryFields; + } + + public List getRelationFields() { + return relationFields; + } + + public void setRelationFields(List relationFields) { + this.relationFields = relationFields; + } + + public List getRelationTableFields() { + if (relationTableFields == null) { + synchronized (SchemaItem.class) { + if (relationTableFields == null) { + relationTableFields = new ArrayList<>(); + getRelationFields().forEach(relationFieldsPair -> { + FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); + FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + if (getAlias().equals(leftFieldItem.getOwner())) { + relationTableFields.add(leftFieldItem); + } else if (getAlias().equals(rightFieldItem.getOwner())) { + relationTableFields.add(rightFieldItem); + } + }); + } + } + } + return relationTableFields; + } + + public List getRelationSelectFields() { + if (relationSelectFieldItem == null) { + synchronized (SchemaItem.class) { + if (relationSelectFieldItem == null) { + relationSelectFieldItem = new ArrayList<>(); + getRelationFields().forEach(relationFieldsPair -> { + FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); + List selectFieldItem = getSchemaItem().getColumnFields() + .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationSelectFieldItem.addAll(selectFieldItem); + } else { + FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + selectFieldItem = getSchemaItem().getColumnFields() + .get(rightFieldItem.getOwner() + "." + rightFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationSelectFieldItem.addAll(selectFieldItem); + } else { + throw new UnsupportedOperationException( + "Relation condition column must in select columns."); + } + } + }); + } + } + } + return relationSelectFieldItem; + } + } + + public static class RelationFieldsPair { + + private FieldItem leftFieldItem; + private FieldItem rightFieldItem; + + public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){ + this.leftFieldItem = leftFieldItem; + this.rightFieldItem = rightFieldItem; + } + + public FieldItem getLeftFieldItem() { + return leftFieldItem; + } + + public void setLeftFieldItem(FieldItem leftFieldItem) { + this.leftFieldItem = leftFieldItem; + } + + public FieldItem getRightFieldItem() { + return rightFieldItem; + } + + public void setRightFieldItem(FieldItem rightFieldItem) { + this.rightFieldItem = rightFieldItem; + } + } + + public static class FieldItem { + + private String fieldName; + private List columnItems = new ArrayList<>(); + private List owners = new ArrayList<>(); + + private boolean method; + private boolean binaryOp; + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public List getColumnItems() { + return columnItems; + } + + public void setColumnItems(List columnItems) { + this.columnItems = columnItems; + } + + public boolean isMethod() { + return method; + } + + public void setMethod(boolean method) { + this.method = method; + } + + public boolean isBinaryOp() { + return binaryOp; + } + + public void setBinaryOp(boolean binaryOp) { + this.binaryOp = binaryOp; + } + + public List getOwners() { + return owners; + } + + public void setOwners(List owners) { + this.owners = owners; + } + + public void addColumn(ColumnItem columnItem) { + columnItems.add(columnItem); + } + + public ColumnItem getColumn() { + if (!columnItems.isEmpty()) { + return columnItems.get(0); + } else { + return null; + } + } + + public String getOwner() { + if (!owners.isEmpty()) { + return owners.get(0); + } else { + return null; + } + } + } + + public static class ColumnItem { + + private String owner; + private String columnName; + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java new file mode 100644 index 00000000..78fb93ec --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java @@ -0,0 +1,205 @@ +package com.alibaba.otter.canal.client.adapter.es.config; + +import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd; +import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import com.alibaba.fastsql.sql.SQLUtils; +import com.alibaba.fastsql.sql.ast.SQLExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr; +import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.fastsql.sql.ast.statement.*; +import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; +import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.fastsql.sql.parser.ParserException; +import com.alibaba.fastsql.sql.parser.SQLStatementParser; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.RelationFieldsPair; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; + +/** + * ES同步指定sql格式解析 + * + * @author rewerma 2018-10-26 下午03:45:49 + * @version 1.0.0 + */ +public class SqlParser { + + /** + * 解析sql + * + * @param sql + * @return + */ + public static SchemaItem parse(String sql) { + try { + SQLStatementParser parser = new MySqlStatementParser(sql); + SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement(); + MySqlSelectQueryBlock sqlSelectQueryBlock = (MySqlSelectQueryBlock) statement.getSelect().getQuery(); + + SchemaItem schemaItem = new SchemaItem(); + schemaItem.setSql(SQLUtils.toMySqlString(sqlSelectQueryBlock)); + SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom(); + List tableItems = new ArrayList<>(); + SqlParser.visitSelectTable(schemaItem, sqlTableSource, tableItems, null); + tableItems.forEach(tableItem -> schemaItem.getAliasTableItems().put(tableItem.getAlias(), tableItem)); + + List fieldItems = collectSelectQueryFields(sqlSelectQueryBlock); + fieldItems.forEach(fieldItem -> schemaItem.getSelectFields().put(fieldItem.getFieldName(), fieldItem)); + + schemaItem.init(); + + if (schemaItem.getAliasTableItems().isEmpty() || schemaItem.getSelectFields().isEmpty()) { + throw new ParserException("Parse sql error"); + } + return schemaItem; + } catch (Exception e) { + throw new ParserException(); + } + } + + /** + * 归集字段 + * + * @param sqlSelectQueryBlock + * @return + */ + private static List collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) { + return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> { + FieldItem fieldItem = new FieldItem(); + fieldItem.setFieldName(selectItem.getAlias()); + visitColumn(selectItem.getExpr(), fieldItem); + return fieldItem; + }).collect(Collectors.toList()); + } + + /** + * 解析字段 + * + * @param expr + * @param fieldItem + */ + private static void visitColumn(SQLExpr expr, FieldItem fieldItem) { + if (expr instanceof SQLIdentifierExpr) { + // 无owner + SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr; + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(identifierExpr.getName()); + } + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(identifierExpr.getName()); + fieldItem.getOwners().add(null); + fieldItem.addColumn(columnItem); + } else if (expr instanceof SQLPropertyExpr) { + // 有owner + SQLPropertyExpr sqlPropertyExpr = (SQLPropertyExpr) expr; + if (fieldItem.getFieldName() == null) { + fieldItem.setFieldName(sqlPropertyExpr.getName()); + } + fieldItem.getOwners().add(sqlPropertyExpr.getOwnernName()); + ColumnItem columnItem = new ColumnItem(); + columnItem.setColumnName(sqlPropertyExpr.getName()); + columnItem.setOwner(sqlPropertyExpr.getOwnernName()); + fieldItem.addColumn(columnItem); + } else if (expr instanceof SQLMethodInvokeExpr) { + SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr) expr; + fieldItem.setMethod(true); + for (SQLExpr sqlExpr : methodInvokeExpr.getArguments()) { + visitColumn(sqlExpr, fieldItem); + } + } else if (expr instanceof SQLBinaryOpExpr) { + SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr; + fieldItem.setBinaryOp(true); + visitColumn(sqlBinaryOpExpr.getLeft(), fieldItem); + visitColumn(sqlBinaryOpExpr.getRight(), fieldItem); + } + } + + /** + * 解析表 + */ + private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource, + List tableItems, TableItem tableItemTmp) { + if (sqlTableSource instanceof SQLExprTableSource) { + SQLExprTableSource sqlExprTableSource = (SQLExprTableSource) sqlTableSource; + TableItem tableItem; + if (tableItemTmp != null) { + tableItem = tableItemTmp; + } else { + tableItem = new TableItem(schemaItem); + } + tableItem.setSchema(sqlExprTableSource.getSchema()); + tableItem.setTableName(sqlExprTableSource.getTableName()); + if (tableItem.getAlias() == null) { + tableItem.setAlias(sqlExprTableSource.getAlias()); + } + if (tableItems.isEmpty()) { + // 第一张表为主表 + tableItem.setMain(true); + } + tableItems.add(tableItem); + } else if (sqlTableSource instanceof SQLJoinTableSource) { + SQLJoinTableSource sqlJoinTableSource = (SQLJoinTableSource) sqlTableSource; + SQLTableSource leftTableSource = sqlJoinTableSource.getLeft(); + visitSelectTable(schemaItem, leftTableSource, tableItems, null); + SQLTableSource rightTableSource = sqlJoinTableSource.getRight(); + TableItem rightTableItem = new TableItem(schemaItem); + // 解析on条件字段 + visitOnCondition(sqlJoinTableSource.getCondition(), rightTableItem); + visitSelectTable(schemaItem, rightTableSource, tableItems, rightTableItem); + + } else if (sqlTableSource instanceof SQLSubqueryTableSource) { + SQLSubqueryTableSource subQueryTableSource = (SQLSubqueryTableSource) sqlTableSource; + MySqlSelectQueryBlock sqlSelectQuery = (MySqlSelectQueryBlock) subQueryTableSource.getSelect().getQuery(); + TableItem tableItem; + if (tableItemTmp != null) { + tableItem = tableItemTmp; + } else { + tableItem = new TableItem(schemaItem); + } + tableItem.setAlias(subQueryTableSource.getAlias()); + tableItem.setSubQuerySql(SQLUtils.toMySqlString(sqlSelectQuery)); + tableItem.setSubQuery(true); + tableItem.setSubQueryFields(collectSelectQueryFields(sqlSelectQuery)); + visitSelectTable(schemaItem, sqlSelectQuery.getFrom(), tableItems, tableItem); + } + } + + /** + * 解析on条件 + * + * @param expr + * @param tableItem + */ + private static void visitOnCondition(SQLExpr expr, TableItem tableItem) { + if (!(expr instanceof SQLBinaryOpExpr)) { + throw new UnsupportedOperationException(); + } + SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr; + if (sqlBinaryOpExpr.getOperator() == BooleanAnd) { + visitOnCondition(sqlBinaryOpExpr.getLeft(), tableItem); + visitOnCondition(sqlBinaryOpExpr.getRight(), tableItem); + } else if (sqlBinaryOpExpr.getOperator() == Equality) { + FieldItem leftFieldItem = new FieldItem(); + visitColumn(sqlBinaryOpExpr.getLeft(), leftFieldItem); + if (leftFieldItem.getColumnItems().size() != 1 || leftFieldItem.isMethod() || leftFieldItem.isBinaryOp()) { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + FieldItem rightFieldItem = new FieldItem(); + visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem); + if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() + || rightFieldItem.isBinaryOp()) { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem)); + } else { + throw new UnsupportedOperationException("Unsupported for complex of on-condition"); + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java new file mode 100644 index 00000000..c3eb1e3e --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -0,0 +1,15 @@ +package com.alibaba.otter.canal.client.adapter.es.service; + +import org.elasticsearch.client.transport.TransportClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ESSyncService { + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private TransportClient transportClient; + + public ESSyncService(TransportClient transportClient) { + this.transportClient = transportClient; + } +} diff --git a/client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter b/client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter new file mode 100644 index 00000000..b30310ee --- /dev/null +++ b/client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter @@ -0,0 +1 @@ +es=com.alibaba.otter.canal.client.adapter.es.ESAdapter \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml b/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml new file mode 100644 index 00000000..df6239b6 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml @@ -0,0 +1,12 @@ +dataSourceKey: defaultDS +esMapping: + _index: mytest_user + _type: _doc + _id: id +# pk: id + sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user.a + left join role b on a.role_id=b.id + left join (select user_id, group_concat(label,',') as labels from user_label + group by user_id) c on c.user_id=a.id" + commitBatch: 3000 + etlCondition: "where a.c_time>='{0}' or b.c_time>='{0}' or c.c_time>='{0}'" \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java new file mode 100644 index 00000000..b2586632 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Map; + +public class ConfigLoadTest { + + @Before + public void before() { + AdapterConfigs.put("es", "myetst_user.yml"); + } + + @Test + public void testLoad() { + Map configMap = ESSyncConfigLoader.load(); + ESSyncConfig config = configMap.get("myetst_user.yml"); + Assert.assertNotNull(config); + Assert.assertEquals("defaultDS", config.getDataSourceKey()); + ESSyncConfig.ESMapping esMapping = config.getEsMapping(); + Assert.assertEquals("mytest_user", esMapping.get_index()); + Assert.assertEquals("_doc", esMapping.get_type()); + Assert.assertEquals("id", esMapping.get_id()); + Assert.assertNotNull(esMapping.getSql()); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java new file mode 100644 index 00000000..20025ad5 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java @@ -0,0 +1,51 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.es.config.SqlParser; + +public class SqlParseTest { + + @Test + public void parseTest() { + // String sql = "select a.id,d.user_id2, concat(name,'_', a.nick) as name, + // b.name as roleNme, d.name as typeName,concat(d.label,'_') as label," + // + " c.name as refName from user a left join role b on b.user_id=a.id " + // + "left join type d on d.user_id=a.id and d.user_id2=a.p_id " + // + "left join ( select ref_id,group_concat(name,',') as name from role group + // by ref_id ) c on c.ref_id=a.id " + // + "where a.id=1"; + String sql = "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a " + + "left join role b on a.role_id=b.id " + + "left join (select user_id, group_concat(label,',') as labels from user_label " + + "group by user_id) c on c.user_id=a.id"; + SchemaItem schemaItem = SqlParser.parse(sql); + + // 通过表名找 TableItem + List tableItems = schemaItem.getTableItemAliases().get("user_label".toLowerCase()); + tableItems.forEach(tableItem -> Assert.assertEquals("c", tableItem.getAlias())); + + TableItem tableItem = tableItems.get(0); + Assert.assertFalse(tableItem.isMain()); + Assert.assertTrue(tableItem.isSubQuery()); + // 通过字段名找 FieldItem + List fieldItems = schemaItem.getColumnFields().get(tableItem.getAlias() + ".label".toLowerCase()); + fieldItems.forEach( + fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName())); + + // 获取当前表关联条件字段 + List relationTableFields = tableItem.getRelationTableFields(); + relationTableFields.forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); + + // 获取关联字段在select中的对应字段 + List relationSelectFieldItem = tableItem.getRelationSelectFields(); + relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", + fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); + } +} diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index e2be0d35..b7f66111 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -21,6 +21,7 @@ common logger hbase + elasticsearch launcher From 11ac253c46b7043d59cb8d5ea92f313d22e28b32 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 10:12:19 +0800 Subject: [PATCH 06/42] =?UTF-8?q?adapter=20pom=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/common/pom.xml | 2 +- client-adapter/elasticsearch/pom.xml | 4 ++-- client-adapter/launcher/pom.xml | 26 ++++++++++++++++++++++++-- client-adapter/pom.xml | 3 ++- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index e62fc386..f018c490 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -14,7 +14,7 @@ com.alibaba.otter canal.protocol - 1.1.2-SNAPSHOT + ${canal_version} joda-time diff --git a/client-adapter/elasticsearch/pom.xml b/client-adapter/elasticsearch/pom.xml index b5a8c11b..bff1bffd 100644 --- a/client-adapter/elasticsearch/pom.xml +++ b/client-adapter/elasticsearch/pom.xml @@ -5,13 +5,13 @@ canal.client-adapter com.alibaba.otter - 1.1.1-SNAPSHOT + 1.1.2-SNAPSHOT 4.0.0 com.alibaba.otter client-adapter.elasticsearch jar - canal client adapter ElasticSearch module for otter ${project.version} + canal client adapter elasticsearch module for otter ${project.version} diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 1a79ed21..8a1a4122 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -32,7 +32,7 @@ com.alibaba.otter canal.client - 1.1.2-SNAPSHOT + ${canal_version} org.yaml @@ -93,6 +93,19 @@ jar-with-dependencies true + + com.alibaba.otter + client-adapter.elasticsearch + ${project.version} + + + * + * + + + jar-with-dependencies + true + @@ -104,13 +117,17 @@ 2.0.1.RELEASE + + com.alibaba.otter + client-adapter.logger + com.alibaba.otter client-adapter.hbase com.alibaba.otter - client-adapter.logger + client-adapter.elasticsearch @@ -142,6 +159,11 @@ + + + + + diff --git a/client-adapter/pom.xml b/client-adapter/pom.xml index b7f66111..d18d1c19 100644 --- a/client-adapter/pom.xml +++ b/client-adapter/pom.xml @@ -14,7 +14,8 @@ 1.8 1.8 UTF-8 - 1.1.1-SNAPSHOT + + 1.1.2-SNAPSHOT From 4ce2b94241fcc9fdca6e3c2c97a99a5ecbdb1a8c Mon Sep 17 00:00:00 2001 From: wuwo Date: Tue, 30 Oct 2018 11:49:49 +0800 Subject: [PATCH 07/42] rollback tsdb change --- .../inbound/mysql/dbsync/LogEventConvert.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 5288ac2a..5a302b86 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -233,7 +233,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar if (useDruidDdlFilter) { List results = DruidDdlParser.parse(queryString, event.getDbName()); for (DdlResult result : results) { - if (!processFilter(queryString, result, event, isSeek)) { + if (!processFilter(queryString, result)) { // 只要有一个数据不进行过滤 notFilter = true; } @@ -246,7 +246,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } else { DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName()); - if (!processFilter(queryString, result, event, isSeek)) { + if (!processFilter(queryString, result)) { notFilter = true; } @@ -260,6 +260,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return null; } + if (!isSeek) { + // 使用新的表结构元数据管理方式 + EntryPosition position = createPosition(event.getHeader()); + tableMetaCache.apply(position, event.getDbName(), queryString, null); + } + Header header = createHeader(event.getHeader(), schemaName, tableName, type); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); if (type != EventType.QUERY && type != EventType.INSERT && type != EventType.UPDATE @@ -279,7 +285,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return StringUtils.substringAfter(queryString, type); } - private boolean processFilter(String queryString, DdlResult result, QueryLogEvent event, boolean isSeek) { + private boolean processFilter(String queryString, DdlResult result) { String schemaName = result.getSchemaName(); String tableName = result.getTableName(); // fixed issue https://github.com/alibaba/canal/issues/58 @@ -306,6 +312,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) { // 针对DDL类型 + if (filterQueryDdl) { + return true; + } + if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) { // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析 @@ -341,16 +351,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } } - - if (!isSeek) { - // 使用新的表结构元数据管理方式 - EntryPosition position = createPosition(event.getHeader()); - tableMetaCache.apply(position, event.getDbName(), queryString, null); - } - - if (filterQueryDdl) { - return true; - } } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) { // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤 From e4e23c9ecd0ca5a07cdb9436f2f7e27b40e8b055 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 14:13:51 +0800 Subject: [PATCH 08/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0adapter=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E5=8F=82=E6=95=B0:=20retry=20timeout=20batchSize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/support/CanalClientConfig.java | 38 ++- .../canal/client/adapter/es/ESAdapter.java | 10 +- .../adapter/es/service/ESSyncService.java | 10 +- .../client/adapter/es/support/ESTemplate.java | 271 ++++++++++++++++++ .../client/adapter/es/test/SqlParseTest.java | 7 - .../loader/AbstractCanalAdapterWorker.java | 53 +++- .../loader/CanalAdapterKafkaWorker.java | 54 ++-- .../launcher/loader/CanalAdapterLoader.java | 16 +- .../loader/CanalAdapterRocketMQWorker.java | 49 ++-- .../launcher/loader/CanalAdapterWorker.java | 96 ++++--- .../src/main/resources/application.yml | 31 +- .../client/impl/SimpleCanalConnector.java | 6 +- .../client/kafka/KafkaCanalConnector.java | 8 +- .../client/kafka/KafkaCanalConnectors.java | 50 ---- .../rocketmq/RocketMQCanalConnectors.java | 24 -- .../kafka/CanalKafkaClientExample.java | 6 +- .../running/kafka/KafkaClientRunningTest.java | 2 +- .../rocketmq/CanalRocketMQClientExample.java | 3 +- 18 files changed, 513 insertions(+), 221 deletions(-) create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java delete mode 100644 client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java delete mode 100644 client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java index f08c6435..3d16a067 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java @@ -19,6 +19,12 @@ public class CanalClientConfig { private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式 + private Integer batchSize; // 批大小 + + private Integer retry; // 重试次数 + + private Long timeout; // 消费超时时间 + private List mqTopics; // mq topic 列表 private List canalInstances; // tcp 模式下 canal 实例列表, 与mq模式不能共存!! @@ -63,6 +69,30 @@ public class CanalClientConfig { this.flatMessage = flatMessage; } + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + + public Integer getRetry() { + return retry; + } + + public void setRetry(Integer retry) { + this.retry = retry; + } + + public Long getTimeout() { + return timeout; + } + + public void setTimeout(Long timeout) { + this.timeout = timeout; + } + public List getCanalInstances() { return canalInstances; } @@ -73,9 +103,9 @@ public class CanalClientConfig { public static class CanalInstance { - private String instance; // 实例名 + private String instance; // 实例名 - private List groups; // 适配器分组列表 + private List groups; // 适配器分组列表 public String getInstance() { return instance; @@ -112,9 +142,9 @@ public class CanalClientConfig { public static class MQTopic { - private String mqMode; // mq模式 kafka or rocketMQ + private String mqMode; // mq模式 kafka or rocketMQ - private String topic; // topic名 + private String topic; // topic名 private List groups = new ArrayList<>(); // 分组列表 diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index 8e2523e4..2862c5c9 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -3,7 +3,6 @@ package com.alibaba.otter.canal.client.adapter.es; import java.util.List; import java.util.Map; -import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.transport.client.PreBuiltTransportClient; @@ -13,6 +12,8 @@ import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; @@ -50,7 +51,8 @@ public class ESAdapter implements OuterAdapter { properties.forEach(settingBuilder::put); Settings settings = settingBuilder.build(); transportClient = new PreBuiltTransportClient(settings); - esSyncService = new ESSyncService(transportClient); + ESTemplate esTemplate = new ESTemplate(transportClient); + esSyncService = new ESSyncService(esTemplate); } catch (Exception e) { throw new RuntimeException(e); } @@ -72,7 +74,9 @@ public class ESAdapter implements OuterAdapter { @Override public void destroy() { - + if (transportClient != null) { + transportClient.close(); + } } @Override diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index c3eb1e3e..2dd68abd 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -1,15 +1,17 @@ package com.alibaba.otter.canal.client.adapter.es.service; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import org.elasticsearch.client.transport.TransportClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ESSyncService { - private Logger logger = LoggerFactory.getLogger(this.getClass()); - private TransportClient transportClient; + private static Logger logger = LoggerFactory.getLogger(ESSyncService.class); - public ESSyncService(TransportClient transportClient) { - this.transportClient = transportClient; + private ESTemplate esTemplate; + + public ESSyncService(ESTemplate esTemplate){ + this.esTemplate = esTemplate; } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java new file mode 100644 index 00000000..4950a41c --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -0,0 +1,271 @@ +package com.alibaba.otter.canal.client.adapter.es.support; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.UpdateByQueryAction; +import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; + +public class ESTemplate { + + private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); + + private static final int MAX_BATCH_SIZE = 1000; + + private TransportClient transportClient; + + public ESTemplate(TransportClient transportClient){ + this.transportClient = transportClient; + } + + public void setTransportClient(TransportClient transportClient) { + this.transportClient = transportClient; + } + + /** + * 插入数据 + * + * @param esSyncConfig + * @param pkVal + * @param esFieldData + * @return + */ + public boolean insert(ESSyncConfig esSyncConfig, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + ESMapping mapping = esSyncConfig.getEsMapping(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setSource(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData)); + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 根据主键更新数据 + * + * @param esSyncConfig + * @param pkVal + * @param esFieldData + * @return + */ + public boolean update(ESSyncConfig esSyncConfig, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + append4Update(bulkRequestBuilder, esSyncConfig, pkVal, esFieldData); + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 结合 addBulkRequest4Update 批量更新 + * + * @param consumer + * @return + */ + public boolean updateBatch(Consumer consumer) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + consumer.accept(bulkRequestBuilder); + return commitBulkRequest(bulkRequestBuilder); + } + + public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESSyncConfig esSyncConfig, Object pkVal, + Map esFieldData) { + ESMapping mapping = esSyncConfig.getEsMapping(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setDoc(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { // 理论上只有一条 + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) + .setDoc(esFieldData)); + } + } + } + + /** + * update by query + * + * @param esSyncConfig + * @param queryBuilder + * @param esFieldData + * @return + */ + private boolean updateByQuery(ESSyncConfig esSyncConfig, QueryBuilder queryBuilder, + Map esFieldData) { + return updateByQuery(esSyncConfig, queryBuilder, esFieldData, 0); + } + + private boolean updateByQuery(ESSyncConfig esSyncConfig, QueryBuilder queryBuilder, Map esFieldData, + int counter) { + if (CollectionUtils.isEmpty(esFieldData)) { + return true; + } + + ESMapping mapping = esSyncConfig.getEsMapping(); + + StringBuilder sb = new StringBuilder(); + esFieldData.forEach((key, value) -> { + if (value instanceof Map) { + HashMap mapValue = (HashMap) value; + if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = [") + .append(mapValue.get("lon")) + .append(", ") + .append(mapValue.get("lat")) + .append("];"); + } else { + logger.warn("Unsupported object type for script_update"); + } + } else if (value instanceof String) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = '") + .append(value) + .append("';"); + } else { + sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";"); + } + }); + String scriptLine = sb.toString(); + if (logger.isDebugEnabled()) { + logger.debug(scriptLine); + } + + UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); + updateByQuery.source(mapping.get_index()) + .abortOnVersionConflict(false) + .filter(queryBuilder) + .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); + + BulkByScrollResponse response = updateByQuery.get(); + if (logger.isDebugEnabled()) { + logger.debug("updateByQuery response: {}", response.getStatus()); + } + if (!CollectionUtils.isEmpty(response.getSearchFailures())) { + logger.error("script update_for_search has search error: " + response.getBulkFailures()); + return false; + } + + if (!CollectionUtils.isEmpty(response.getBulkFailures())) { + logger.error("script update_for_search has update error: " + response.getBulkFailures()); + return false; + } + + if (response.getStatus().getVersionConflicts() > 0) { + if (counter >= 3) { + logger.error("第 {} 次执行updateByQuery, 依旧存在版本冲突，不再继续重试。", counter); + return false; + } + logger.warn("本次updateByQuery存在版本冲突，准备重新执行..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + // ignore + } + return updateByQuery(esSyncConfig, queryBuilder, esFieldData, ++counter); + } + + return true; + } + + /** + * 通过主键删除数据 + * + * @param esSyncConfig + * @param pkVal + * @return + */ + public boolean delete(ESSyncConfig esSyncConfig, Object pkVal) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + ESMapping mapping = esSyncConfig.getEsMapping(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 批量提交 + * + * @param bulkRequestBuilder + * @return + */ + private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { + if (bulkRequestBuilder.numberOfActions() > 0) { + BulkResponse response = bulkRequestBuilder.execute().actionGet(); + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.warn(itemResponse.getFailureMessage()); + } else { + logger.error("ES 同步数据错误 {}", itemResponse.getFailureMessage()); + } + } + } + + return !response.hasFailures(); + } + + return true; + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java index 20025ad5..6d0644d2 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java @@ -14,13 +14,6 @@ public class SqlParseTest { @Test public void parseTest() { - // String sql = "select a.id,d.user_id2, concat(name,'_', a.nick) as name, - // b.name as roleNme, d.name as typeName,concat(d.label,'_') as label," - // + " c.name as refName from user a left join role b on b.user_id=a.id " - // + "left join type d on d.user_id=a.id and d.user_id2=a.p_id " - // + "left join ( select ref_id,group_concat(name,',') as name from role group - // by ref_id ) c on c.ref_id=a.id " - // + "where a.id=1"; String sql = "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a " + "left join role b on a.role_id=b.id " + "left join (select user_id, group_concat(label,',') as labels from user_label " diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index a5a7a730..9e7adeca 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -2,11 +2,10 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalMQConnector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,6 +116,52 @@ public abstract class AbstractCanalAdapterWorker { }); } + protected void mqWriteOutData(int retry, long timeout, boolean flatMessage, CanalMQConnector connector, + ExecutorService workerExecutor) { + for (int i = 0; i < retry; i++) { + try { + List messages; + if (!flatMessage) { + messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); + } else { + messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS); + } + if (messages != null) { + Future future = workerExecutor.submit(() -> { + for (final Object message : messages) { + if (message instanceof FlatMessage) { + writeOut((FlatMessage) message); + } else { + writeOut((Message) message); + } + } + return true; + }); + + try { + future.get(timeout, TimeUnit.MILLISECONDS); + } catch (Exception e) { + future.cancel(true); + throw e; + } + } + connector.ack(); + break; + } catch (Throwable e) { + if (i == retry - 1) { + connector.ack(); + } + + logger.error(e.getMessage(), e); + try { + TimeUnit.SECONDS.sleep(1L); + } catch (InterruptedException e1) { + // ignore + } + } + } + } + public void start() { if (!running) { thread = new Thread(this::process); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java index 4aa5aee4..b8de5269 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -1,14 +1,14 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.util.List; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import org.apache.kafka.clients.consumer.CommitFailedException; import org.apache.kafka.common.errors.WakeupException; import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; @@ -20,17 +20,24 @@ import com.alibaba.otter.canal.protocol.Message; */ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { + private CanalClientConfig canalClientConfig; private KafkaCanalConnector connector; private String topic; private boolean flatMessage; - public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId, - List> canalOuterAdapters, boolean flatMessage){ + public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic, + String groupId, List> canalOuterAdapters, boolean flatMessage){ super(canalOuterAdapters); + this.canalClientConfig = canalClientConfig; this.topic = topic; this.canalDestination = topic; this.flatMessage = flatMessage; - this.connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage); + this.connector = new KafkaCanalConnector(bootstrapServers, + topic, + null, + groupId, + canalClientConfig.getBatchSize(), + flatMessage); // connector.setSessionTimeout(1L, TimeUnit.MINUTES); } @@ -38,6 +45,10 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { protected void process() { while (!running) ; + ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); + int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); + long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 + while (running) { try { syncSwitch.get(canalDestination); @@ -47,35 +58,12 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { connector.subscribe(); logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); while (running) { - try { - Boolean status = syncSwitch.status(canalDestination); - if (status != null && !status) { - connector.disconnect(); - break; - } - - List messages; - if (!flatMessage) { - messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); - } else { - messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS); - } - if (messages != null) { - for (final Object message : messages) { - if (message instanceof FlatMessage) { - writeOut((FlatMessage) message); - } else { - writeOut((Message) message); - } - } - } - connector.ack(); - } catch (CommitFailedException e) { - logger.warn(e.getMessage()); - } catch (Throwable e) { - logger.error(e.getMessage(), e); - TimeUnit.SECONDS.sleep(1L); + Boolean status = syncSwitch.status(canalDestination); + if (status != null && !status) { + connector.disconnect(); + break; } + mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor); } } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 627ca8f6..df641c45 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -67,9 +67,15 @@ public class CanalAdapterLoader { } CanalAdapterWorker worker; if (sa != null) { - worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups); + worker = new CanalAdapterWorker(canalClientConfig, + instance.getInstance(), + sa, + canalOuterAdapterGroups); } else if (zkHosts != null) { - worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups); + worker = new CanalAdapterWorker(canalClientConfig, + instance.getInstance(), + zkHosts, + canalOuterAdapterGroups); } else { throw new RuntimeException("No canal server connector found"); } @@ -90,7 +96,8 @@ public class CanalAdapterLoader { } canalOuterAdapterGroups.add(canalOuterAdapters); if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) { - CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig.getBootstrapServers(), + CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, + canalClientConfig.getBootstrapServers(), topic.getTopic(), group.getGroupId(), canalOuterAdapterGroups, @@ -98,7 +105,8 @@ public class CanalAdapterLoader { canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); rocketMQWorker.start(); } else if ("kafka".equalsIgnoreCase(topic.getMqMode())) { - CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig.getBootstrapServers(), + CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, + canalClientConfig.getBootstrapServers(), topic.getTopic(), group.getGroupId(), canalOuterAdapterGroups, diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java index 69862f88..133e4919 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -1,13 +1,16 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import org.apache.kafka.common.errors.WakeupException; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; -import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectors; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; @@ -18,17 +21,19 @@ import com.alibaba.otter.canal.protocol.Message; */ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { + private CanalClientConfig canalClientConfig; private RocketMQCanalConnector connector; private String topic; private boolean flatMessage; - public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId, - List> canalOuterAdapters, boolean flatMessage){ + public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic, + String groupId, List> canalOuterAdapters, boolean flatMessage){ super(canalOuterAdapters); + this.canalClientConfig = canalClientConfig; this.topic = topic; this.flatMessage = flatMessage; this.canalDestination = topic; - this.connector = RocketMQCanalConnectors.newRocketMQConnector(nameServers, topic, groupId, flatMessage); + this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage); logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId); } @@ -36,6 +41,11 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { protected void process() { while (!running) ; + + ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); + int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); + long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 + while (running) { try { syncSwitch.get(canalDestination); @@ -45,33 +55,12 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { connector.subscribe(); logger.info("=============> Subscribe topic: {} succeed<=============", this.topic); while (running) { - try { - Boolean status = syncSwitch.status(canalDestination); - if (status != null && !status) { - connector.disconnect(); - break; - } - - List messages; - if (!flatMessage) { - messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); - } else { - messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS); - } - if (messages != null) { - for (final Object message : messages) { - if (message instanceof FlatMessage) { - writeOut((FlatMessage) message); - } else { - writeOut((Message) message); - } - } - } - connector.ack(); - } catch (Throwable e) { - logger.error(e.getMessage(), e); - TimeUnit.SECONDS.sleep(1L); + Boolean status = syncSwitch.status(canalDestination); + if (status != null && !status) { + connector.disconnect(); + break; } + mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor); } } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java index ec09da6a..52b4e1af 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java @@ -2,12 +2,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.net.SocketAddress; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.*; import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnectors; import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; import com.alibaba.otter.canal.protocol.Message; @@ -20,10 +20,12 @@ import com.alibaba.otter.canal.protocol.Message; */ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { - private static final int BATCH_SIZE = 50; - private static final int SO_TIMEOUT = 0; + private static final int BATCH_SIZE = 50; + private static final int SO_TIMEOUT = 0; - private CanalConnector connector; + private CanalConnector connector; + + private CanalClientConfig canalClientConfig; /** * 单台client适配器worker的构造方法 @@ -32,9 +34,10 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { * @param address canal-server地址 * @param canalOuterAdapters 外部适配器组 */ - public CanalAdapterWorker(String canalDestination, SocketAddress address, + public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address, List> canalOuterAdapters){ super(canalOuterAdapters); + this.canalClientConfig = canalClientConfig; this.canalDestination = canalDestination; connector = CanalConnectors.newSingleConnector(address, canalDestination, "", ""); } @@ -46,10 +49,11 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { * @param zookeeperHosts zookeeper地址 * @param canalOuterAdapters 外部适配器组 */ - public CanalAdapterWorker(String canalDestination, String zookeeperHosts, + public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts, List> canalOuterAdapters){ super(canalOuterAdapters); this.canalDestination = canalDestination; + this.canalClientConfig = canalClientConfig; connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", ""); ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT); } @@ -58,6 +62,15 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { protected void process() { while (!running) ; // waiting until running == true + + ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); + int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); + long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟 + Integer batchSize = canalClientConfig.getBatchSize(); + if (batchSize == null) { + batchSize = BATCH_SIZE; + } + while (running) { try { syncSwitch.get(canalDestination); @@ -77,35 +90,50 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { break; } - // server配置canal.instance.network.soTimeout(默认: 30s) - // 范围内未与server交互，server将关闭本次socket连接 - Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据 - long batchId = message.getId(); - try { - int size = message.getEntries().size(); - if (batchId == -1 || size == 0) { - Thread.sleep(1000); - } else { - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} batchSize: {} ", - this.canalDestination, - batchId, - size); + for (int i = 0; i < retry; i++) { + Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 + long batchId = message.getId(); + try { + int size = message.getEntries().size(); + if (batchId == -1 || size == 0) { + Thread.sleep(500); + } else { + Future future = workerExecutor.submit(() -> { + if (logger.isDebugEnabled()) { + logger.debug("destination: {} batchId: {} batchSize: {} ", + canalDestination, + batchId, + size); + } + long begin = System.currentTimeMillis(); + writeOut(message); + if (logger.isDebugEnabled()) { + logger.debug("destination: {} batchId: {} elapsed time: {} ms", + canalDestination, + batchId, + System.currentTimeMillis() - begin); + } + return true; + }); + + try { + future.get(timeout, TimeUnit.MILLISECONDS); + } catch (Exception e) { + future.cancel(true); + throw e; + } } - long begin = System.currentTimeMillis(); - writeOut(message); - if (logger.isDebugEnabled()) { - logger.debug("destination: {} batchId: {} elapsed time: {} ms", - this.canalDestination, - batchId, - System.currentTimeMillis() - begin); + connector.ack(batchId); // 提交确认 + break; + } catch (Exception e) { + if (i != retry - 1) { + connector.rollback(batchId); // 处理失败, 回滚数据 + } else { + connector.ack(batchId); } + logger.error("sync error!", e); + Thread.sleep(500); } - connector.ack(batchId); // 提交确认 - } catch (Exception e) { - connector.rollback(batchId); // 处理失败, 回滚数据 - logger.error("sync error!", e); - Thread.sleep(500); } } @@ -124,6 +152,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker { } } } + + workerExecutor.shutdown(); } @Override diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 250d6a60..26cab04c 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -15,27 +15,30 @@ hbase.zookeeper.property.clientPort: 2181 hbase.zookeeper.znode.parent: /hbase canal.conf: - canalServerHost: 127.0.0.1:11111 +# canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 -# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 + bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 flatMessage: true - canalInstances: - - instance: example - groups: - - outAdapters: - - name: logger + retry: 3 + timeout: 20000 + batchSize: 50 +# canalInstances: +# - instance: example +# groups: +# - outAdapters: +# - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} # hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} # zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} -# mqTopics: -# - mqMode: kafka -# topic: example -# groups: -# - groupId: g2 -# outAdapters: -# - name: logger + mqTopics: + - mqMode: kafka + topic: example + groups: + - groupId: g2 + outAdapters: + - name: logger # mqTopics: # - mqMode: rocketmq # topic: example diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java index 25ad5056..4c2211d2 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java @@ -133,7 +133,7 @@ public class SimpleCanalConnector implements CanalConnector { runningMonitor.stop(); } } else { - doDisconnnect(); + doDisconnect(); } } @@ -190,7 +190,7 @@ public class SimpleCanalConnector implements CanalConnector { } } - private void doDisconnnect() throws CanalClientException { + private void doDisconnect() throws CanalClientException { if (readableChannel != null) { quietlyClose(readableChannel); readableChannel = null; @@ -434,7 +434,7 @@ public class SimpleCanalConnector implements CanalConnector { public void processActiveExit() { mutex.set(false); - doDisconnnect(); + doDisconnect(); } }); diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java index bf652ecf..31f75b19 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java @@ -42,7 +42,8 @@ public class KafkaCanalConnector implements CanalMQConnector { private volatile boolean running = false; private boolean flatMessage; - public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, boolean flatMessage){ + public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize, + boolean flatMessage){ this.topic = topic; this.partition = partition; this.flatMessage = flatMessage; @@ -55,7 +56,10 @@ public class KafkaCanalConnector implements CanalMQConnector { properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读 properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 properties.put("session.timeout.ms", "30000"); // 默认为30秒 - properties.put("max.poll.records", "100"); + if (batchSize == null) { + batchSize = 100; + } + properties.put("max.poll.records", batchSize.toString()); properties.put("key.deserializer", StringDeserializer.class.getName()); if (!flatMessage) { properties.put("value.deserializer", MessageDeserializer.class.getName()); diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java deleted file mode 100644 index 950cdcfd..00000000 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.alibaba.otter.canal.client.kafka; - -/** - * canal kafka connectors创建工具类 - * - * @author machengyuan @ 2018-6-12 - * @version 1.0.0 - */ -public class KafkaCanalConnectors { - - /** - * 创建kafka客户端链接，独立运行不注册zk信息 - * - * @param servers - * @param topic - * @param partition - * @param groupId - * @return - */ - public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) { - return new KafkaCanalConnector(servers, topic, partition, groupId, false); - } - - /** - * 创建kafka客户端链接，独立运行不注册zk信息 - * - * @param servers - * @param topic - * @param groupId - * @return - */ - public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) { - return new KafkaCanalConnector(servers, topic, null, groupId, false); - } - - /** - * 创建kafka客户端链接 - * - * @param servers - * @param topic - * @param partition - * @param groupId - * @param flatMessage - * @return - */ - public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, - String groupId, boolean flatMessage) { - return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage); - } -} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java b/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java deleted file mode 100644 index 6dadd44f..00000000 --- a/client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.alibaba.otter.canal.client.rocketmq; - -/** - * RocketMQ connector provider. - */ -public class RocketMQCanalConnectors { - - /** - * Create RocketMQ connector - * - * @param nameServers name servers for RocketMQ - * @param topic - * @param groupId - * @return {@link RocketMQCanalConnector} - */ - public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId) { - return new RocketMQCanalConnector(nameServers, topic, groupId, false); - } - - public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId, - boolean flatMessage) { - return new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage); - } -} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java index 25fefe38..c72b229d 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java @@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; import com.alibaba.otter.canal.protocol.Message; /** @@ -36,12 +35,13 @@ public class CanalKafkaClientExample { }; public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){ - connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false); + connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false); } public static void main(String[] args) { try { - final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers, + final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample( + AbstractKafkaTest.zkServers, AbstractKafkaTest.servers, AbstractKafkaTest.topic, AbstractKafkaTest.partition, diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java index 1b2afec4..96fceb1d 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java @@ -30,7 +30,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest { public void testKafkaConsumer() { final ExecutorService executor = Executors.newFixedThreadPool(1); - final KafkaCanalConnector connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId); + final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false); executor.submit(new Runnable() { diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java b/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java index 404fa8a0..08d53116 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java @@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; -import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectors; import com.alibaba.otter.canal.client.running.kafka.AbstractKafkaTest; import com.alibaba.otter.canal.protocol.Message; @@ -37,7 +36,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest { }; public CanalRocketMQClientExample(String nameServers, String topic, String groupId){ - connector = RocketMQCanalConnectors.newRocketMQConnector(nameServers, topic, groupId); + connector = new RocketMQCanalConnector(nameServers, topic, groupId, false); } public static void main(String[] args) { From e1191b665e163f7d385fcce55f8876b1b1b85fa2 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 15:57:00 +0800 Subject: [PATCH 09/42] =?UTF-8?q?es=E9=85=8D=E7=BD=AE=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 17 ++---- .../adapter/es/config/ESSyncConfig.java | 9 +++ .../adapter/es/config/ESSyncConfigLoader.java | 57 ++++++++++++++----- .../src/main/resources/es/myetst_user.yml | 2 +- .../adapter/es/test/ConfigLoadTest.java | 22 +++++-- .../adapter/es/test/DataSourceConstant.java | 31 ++++++++++ .../running/kafka/KafkaClientRunningTest.java | 1 - 7 files changed, 105 insertions(+), 34 deletions(-) create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index 2862c5c9..f3cdf6d3 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -28,24 +28,17 @@ import com.alibaba.otter.canal.client.adapter.support.SPI; @SPI("es") public class ESAdapter implements OuterAdapter { - private static Logger logger = LoggerFactory.getLogger(ESAdapter.class); + private static Logger logger = LoggerFactory.getLogger(ESAdapter.class); - private static volatile Map esSyncConfig = null; // 文件名对应配置 + private TransportClient transportClient; - private TransportClient transportClient; - - private ESSyncService esSyncService; + private ESSyncService esSyncService; @Override public void init(OuterAdapterConfig configuration) { try { - if (esSyncConfig == null) { - synchronized (ESSyncConfig.class) { - if (esSyncConfig == null) { - esSyncConfig = ESSyncConfigLoader.load(); - } - } - } + ESSyncConfigLoader.load(); + Map properties = configuration.getProperties(); Settings.Builder settingBuilder = Settings.builder(); properties.forEach(settingBuilder::put); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java index be0b712b..c75fa0bd 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -53,6 +53,8 @@ public class ESSyncConfig { private int commitBatch = 1000; private String etlCondition; + private SchemaItem schemaItem; // sql解析结果模型 + public String get_index() { return _index; } @@ -133,5 +135,12 @@ public class ESSyncConfig { this.etlCondition = etlCondition; } + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index 7e156409..34214e04 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -5,27 +5,38 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; +import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; public class ESSyncConfigLoader { - private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); + private static Logger logger = LoggerFactory + .getLogger(ESSyncConfigLoader.class); - private static final String BASE_PATH = "es"; + private static final String BASE_PATH = "es"; - public static Map load() { + private static volatile Map esSyncConfig = new LinkedHashMap<>(); // 文件名对应配置 + private static volatile Map> dbTableEsSyncConfig = new LinkedHashMap<>(); // schema-table对应配置 + + public static Map getEsSyncConfig() { + return esSyncConfig; + } + + public static Map> getDbTableEsSyncConfig() { + return dbTableEsSyncConfig; + } + + public static synchronized void load() { logger.info("## Start loading mapping config ... "); - - Map result = new LinkedHashMap<>(); - Collection configs = AdapterConfigs.get("es"); for (String c : configs) { if (c == null) { @@ -47,17 +58,35 @@ public class ESSyncConfigLoader { try { config.validate(); + SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql()); + config.getEsMapping().setSchemaItem(schemaItem); + + DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (dataSource == null || dataSource.getUrl() == null) { + throw new RuntimeException("No data source found: " + config.getDataSourceKey()); + } + Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); + Matcher matcher = pattern.matcher(dataSource.getUrl()); + if (!matcher.find()){ + throw new RuntimeException("No found the schema of jdbc-url: " + config.getDataSourceKey()); + } + String schema = matcher.group(2); + + schemaItem.getAliasTableItems().values().forEach(tableItem -> { + List esSyncConfigs = dbTableEsSyncConfig + .computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new ArrayList<>()); + esSyncConfigs.add(config); + }); } catch (Exception e) { - throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e); + throw new RuntimeException("ERROR Config: " + c, e); } - result.put(c, config); + esSyncConfig.put(c, config); } logger.info("## Mapping config loaded"); - return result; } - public static String readConfigContent(String config) { + private static String readConfigContent(String config) { InputStream in = null; try { // 先取本地文件，再取类路径 @@ -68,7 +97,7 @@ public class ESSyncConfigLoader { in = ESSyncConfigLoader.class.getClassLoader().getResourceAsStream(config); } if (in == null) { - throw new RuntimeException("Config file not found."); + throw new RuntimeException("Config file: " + config + " not found."); } byte[] bytes = new byte[in.available()]; diff --git a/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml b/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml index df6239b6..d2e38c46 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml @@ -4,7 +4,7 @@ esMapping: _type: _doc _id: id # pk: id - sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user.a + sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a left join role b on a.role_id=b.id left join (select user_id, group_concat(label,',') as labels from user_label group by user_id) c on c.user_id=a.id" diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java index b2586632..c2a4761e 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -1,24 +1,31 @@ package com.alibaba.otter.canal.client.adapter.es.test; -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; -import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.util.Map; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; public class ConfigLoadTest { @Before - public void before() { + public void before() throws SQLException { AdapterConfigs.put("es", "myetst_user.yml"); + // 加载数据源连接池 + DatasourceConfig.DATA_SOURCES.put("defaultDS", DataSourceConstant.dataSource); } @Test public void testLoad() { - Map configMap = ESSyncConfigLoader.load(); + ESSyncConfigLoader.load(); + Map configMap = ESSyncConfigLoader.getEsSyncConfig(); ESSyncConfig config = configMap.get("myetst_user.yml"); Assert.assertNotNull(config); Assert.assertEquals("defaultDS", config.getDataSourceKey()); @@ -27,5 +34,8 @@ public class ConfigLoadTest { Assert.assertEquals("_doc", esMapping.get_type()); Assert.assertEquals("id", esMapping.get_id()); Assert.assertNotNull(esMapping.getSql()); + + Map> dbTableEsSyncConfig = ESSyncConfigLoader.getDbTableEsSyncConfig(); + Assert.assertFalse(dbTableEsSyncConfig.isEmpty()); } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java new file mode 100644 index 00000000..c29953cd --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import java.sql.SQLException; + +import com.alibaba.druid.pool.DruidDataSource; + +public class DataSourceConstant { + + static DruidDataSource dataSource; + static { + dataSource = new DruidDataSource(); + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl("jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"); + dataSource.setUsername("root"); + dataSource.setPassword("121212"); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(3); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setPoolPreparedStatements(false); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + dataSource.setValidationQuery("select 1"); + try { + dataSource.init(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java index 96fceb1d..9133f74c 100644 --- a/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java @@ -11,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors; import com.alibaba.otter.canal.protocol.Message; /** From 2c9acb655d657c5cf0b4d9addb71eee7b27a1776 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 30 Oct 2018 19:52:05 +0800 Subject: [PATCH 10/42] =?UTF-8?q?=E5=8D=95=E8=A1=A8=E6=96=B0=E5=A2=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 10 +- .../adapter/es/config/ESSyncConfig.java | 10 + .../adapter/es/config/ESSyncConfigLoader.java | 2 +- .../client/adapter/es/config/SchemaItem.java | 29 ++ .../adapter/es/service/ESSyncService.java | 172 +++++++++- .../client/adapter/es/support/ESSyncUtil.java | 293 ++++++++++++++++++ .../client/adapter/es/support/ESTemplate.java | 96 +++++- .../canal/client/adapter/es/support/Util.java | 18 ++ .../src/main/resources/es/myetst_user.yml | 12 - .../src/main/resources/es/mytest_user.yml | 14 + .../adapter/es/test/ConfigLoadTest.java | 7 +- .../client/adapter/es/test/SyncTest.java | 54 ++++ .../client/adapter/hbase/HbaseAdapter.java | 4 +- .../adapter/hbase/config/MappingConfig.java | 19 +- .../main/resources/hbase/mytest_person2.yml | 2 +- .../src/main/resources/application.yml | 46 +-- 16 files changed, 723 insertions(+), 65 deletions(-) create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java delete mode 100644 client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml create mode 100644 client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index f3cdf6d3..983dedc9 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -1,16 +1,17 @@ package com.alibaba.otter.canal.client.adapter.es; +import java.net.InetAddress; import java.util.List; import java.util.Map; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; @@ -44,6 +45,12 @@ public class ESAdapter implements OuterAdapter { properties.forEach(settingBuilder::put); Settings settings = settingBuilder.build(); transportClient = new PreBuiltTransportClient(settings); + String[] hostArray = configuration.getHosts().split(","); + for (String host : hostArray) { + int i = host.indexOf(":"); + transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)), + Integer.parseInt(host.substring(i + 1)))); + } ESTemplate esTemplate = new ESTemplate(transportClient); esSyncService = new ESSyncService(esTemplate); } catch (Exception e) { @@ -53,6 +60,7 @@ public class ESAdapter implements OuterAdapter { @Override public void sync(Dml dml) { + esSyncService.sync(dml); } @Override diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java index c75fa0bd..440ecb6a 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -7,6 +7,8 @@ public class ESSyncConfig { private String dataSourceKey; // 数据源key + private String destination; // canal destination + private ESMapping esMapping; public void validate() { @@ -32,6 +34,14 @@ public class ESSyncConfig { this.dataSourceKey = dataSourceKey; } + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + public ESMapping getEsMapping() { return esMapping; } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index 34214e04..7a54cccd 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -68,7 +68,7 @@ public class ESSyncConfigLoader { Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); Matcher matcher = pattern.matcher(dataSource.getUrl()); if (!matcher.find()){ - throw new RuntimeException("No found the schema of jdbc-url: " + config.getDataSourceKey()); + throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); } String schema = matcher.group(2); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java index 88e29eb6..aa77f1ed 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; public class SchemaItem { @@ -13,10 +14,12 @@ public class SchemaItem { private volatile Map> tableItemAliases; private volatile Map> columnFields; + private volatile Boolean allFieldsSimple; public void init() { this.getTableItemAliases(); this.getColumnFields(); + this.isAllFieldsSimple(); aliasTableItems.values().forEach(tableItem -> { tableItem.getRelationTableFields(); tableItem.getRelationSelectFields(); @@ -89,7 +92,25 @@ public class SchemaItem { } } return columnFields; + } + public boolean isAllFieldsSimple() { + if (allFieldsSimple == null) { + synchronized (SchemaItem.class) { + if (allFieldsSimple == null) { + allFieldsSimple = true; + + for (FieldItem fieldItem : getSelectFields().values()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + } + } + } + + return allFieldsSimple; } public TableItem getMainTable() { @@ -100,6 +121,14 @@ public class SchemaItem { } } + public FieldItem getIdFieldItem(ESMapping mapping) { + if (mapping.get_id() != null) { + return getSelectFields().get(mapping.get_id()); + } else { + return getSelectFields().get(mapping.getPk()); + } + } + public static class TableItem { private SchemaItem schemaItem; diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 2dd68abd..d028f6ba 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -1,10 +1,26 @@ package com.alibaba.otter.canal.client.adapter.es.service; -import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; -import org.elasticsearch.client.transport.TransportClient; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.es.support.Util; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + public class ESSyncService { private static Logger logger = LoggerFactory.getLogger(ESSyncService.class); @@ -14,4 +30,156 @@ public class ESSyncService { public ESSyncService(ESTemplate esTemplate){ this.esTemplate = esTemplate; } + + public void sync(Dml dml) { + long begin = System.currentTimeMillis(); + String database = dml.getDatabase(); + String table = dml.getTable(); + List esSyncConfigs = ESSyncConfigLoader.getDbTableEsSyncConfig().get(database + "-" + table); + if (esSyncConfigs != null) { + if (logger.isDebugEnabled()) { + logger.debug("Destination: {}, database:{}, table:{}, type:{}, effect index count: {}", + dml.getDestination(), + dml.getDatabase(), + dml.getTable(), + dml.getType(), + esSyncConfigs.size()); + logger.debug(JSON.toJSONString(dml)); + } + + for (ESSyncConfig config : esSyncConfigs) { + logger.info("Prepared to sync index: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + this.sync(config, dml); + logger.info("Sync completed: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } + if (logger.isDebugEnabled()) { + logger.debug("Sync elapsed time: {}, effect index count：{}, destination: {}", + (System.currentTimeMillis() - begin), + esSyncConfigs.size(), + dml.getDestination()); + } + } + } + + public void sync(ESSyncConfig config, Dml dml) { + try { + long begin = System.currentTimeMillis(); + + String type = dml.getType(); + if (type != null && type.equalsIgnoreCase("INSERT")) { + insert(config, dml); + } else if (type != null && type.equalsIgnoreCase("UPDATE")) { + // update(config, dml); + } else if (type != null && type.equalsIgnoreCase("DELETE")) { + // delete(config, dml); + } + + if (logger.isDebugEnabled()) { + logger.debug("Sync elapsed time: {},destination: {}, es index: {}", + (System.currentTimeMillis() - begin), + dml.getDestination(), + config.getEsMapping().get_index()); + } + } catch (Exception e) { + logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml); + logger.error(e.getMessage(), e); + } + } + + private void insert(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + ESMapping mapping = config.getEsMapping(); + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + // ------是否单表 & 所有字段都为简单字段------ + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + Map esFieldData = new HashMap<>(); + + // ------通过dml对象插入ES------ + // 获取所有字段 + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + // 如果是主键字段则不插入 + if (fieldItem.getFieldName().equals(mapping.get_id())) { + continue; + } + Object value = esTemplate.getDataValue(mapping, + data, + Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName())); + String fieldName = Util.cleanColumn(fieldItem.getFieldName()); + if (!mapping.getSkips().contains(fieldName)) { + esFieldData.put(fieldName, esTemplate.convertType(mapping, fieldName, value)); + } + } + + // 取出id值 + Object idVal; + if (mapping.get_id() != null) { + idVal = esTemplate.getDataValue(mapping, + data, + Util.cleanColumn( + schemaItem.getSelectFields().get(mapping.get_id()).getColumn().getColumnName())); + } else { + idVal = esTemplate.getDataValue(mapping, + data, + Util.cleanColumn( + schemaItem.getSelectFields().get(mapping.getPk()).getColumn().getColumnName())); + } + if (logger.isDebugEnabled()) { + logger.debug("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.insert(config, idVal, esFieldData); + if (!result) { + logger.error("Single table insert to es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + return; // 单表插入完成返回 + } + + // ------是否主表------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditaionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); +// ESSyncUtil.sqlRS(ds, sql, rs -> { +// try { +// while (rs.next()) { +// Map esFieldData = new HashMap<>(); +// +// Object idVal = ESSyncUtil.persistEsByRs(transportClient, config, rs, mainTable, esFieldData); +// +// if (logger.isDebugEnabled()) { +// logger.debug("主表insert, 通过执行sql查询相应记录，并insert到Es, id: {}, esFieldValue: {}", idVal, esFieldData); +// } +// boolean result = ESSyncUtil.persist(transportClient, config, idVal, esFieldData, OpType.INSERT); +// if (!result) { +// logger.error("主表insert，通过执行sql查询响应记录，并insert到es, es插入失败，table: {}, index: {}", dml.getTable(), config.getEsSyn().getIndex()); +// } +// } +// } catch (Exception e) { +// throw new RuntimeException(e); +// } +// return 0; +// }); + } + } + } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java new file mode 100644 index 00000000..5cd436f5 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -0,0 +1,293 @@ +package com.alibaba.otter.canal.client.adapter.es.support; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.sql.*; +import java.util.*; +import java.util.Date; +import java.util.function.Function; + +import javax.sql.DataSource; + +import org.apache.commons.codec.binary.Base64; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; + +public class ESSyncUtil { + + private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class); + + /** + * 类型转换为Mapping中对应的类型 + */ + public static Object typeConvert(Object val, String esType) { + if (val == null) { + return null; + } + if (esType == null) { + return val; + } + Object res = null; + if ("integer".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).intValue(); + } else { + res = Integer.parseInt(val.toString()); + } + } else if ("long".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).longValue(); + } else { + res = Long.parseLong(val.toString()); + } + } else if ("short".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).shortValue(); + } else { + res = Short.parseShort(val.toString()); + } + } else if ("byte".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).byteValue(); + } else { + res = Byte.parseByte(val.toString()); + } + } else if ("double".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).doubleValue(); + } else { + res = Double.parseDouble(val.toString()); + } + } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) { + if (val instanceof Number) { + res = ((Number) val).floatValue(); + } else { + res = Float.parseFloat(val.toString()); + } + } else if ("boolean".equals(esType)) { + if (val instanceof Boolean) { + res = val; + } else if (val instanceof Number) { + int v = ((Number) val).intValue(); + res = v != 0; + } else { + res = Boolean.parseBoolean(val.toString()); + } + } else if ("date".equals(esType)) { + if (val instanceof java.sql.Time) { + DateTime dateTime = new DateTime(((java.sql.Time) val).getTime()); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("HH:mm:ss.SSS"); + } else { + res = dateTime.toString("HH:mm:ss"); + } + } else if (val instanceof java.sql.Timestamp) { + DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime()); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } else if (val instanceof java.sql.Date || val instanceof Date) { + DateTime dateTime; + if (val instanceof java.sql.Date) { + dateTime = new DateTime(((java.sql.Date) val).getTime()); + } else { + dateTime = new DateTime(((Date) val).getTime()); + } + if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 + && dateTime.getMillisOfSecond() == 0) { + res = dateTime.toString("yyyy-MM-dd"); + } else { + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } + } else if (val instanceof Long) { + DateTime dateTime = new DateTime(((Long) val).longValue()); + if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 + && dateTime.getMillisOfSecond() == 0) { + res = dateTime.toString("yyyy-MM-dd"); + } else if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } else if (val instanceof String) { + String v = ((String) val).trim(); + if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' ' + && v.charAt(13) == ':' && v.charAt(16) == ':') { + String dt = v.substring(0, 10) + "T" + v.substring(11); + DateTime dateTime = new DateTime(dt); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } + } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') { + DateTime dateTime = new DateTime(v); + res = dateTime.toString("yyyy-MM-dd"); + } + } + } else if ("binary".equals(esType)) { + if (val instanceof byte[]) { + Base64 base64 = new Base64(); + res = base64.encodeAsString((byte[]) val); + } else if (val instanceof Blob) { + byte[] b = blobToBytes((Blob) val); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); + } else if (val instanceof String) { + try { + // 对应canal中的单字节编码 + byte[] b = ((String) val).getBytes("ISO-8859-1"); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); + } catch (UnsupportedEncodingException e) { + logger.error(e.getMessage()); + } + } + } else if ("geo_point".equals(esType)) { + if (!(val instanceof String)) { + logger.error("es type is geo_point, but source type is not String"); + return val; + } + + if (!((String) val).contains(",")) { + logger.error("es type is geo_point, source value not contains ',' spit"); + return val; + } + + String[] point = ((String) val).split(","); + Map location = new HashMap<>(); + location.put("lat", Double.valueOf(point[0].trim())); + location.put("lon", Double.valueOf(point[1].trim())); + return location; + } else if ("object".equals(esType)) { + if ("".equals(val.toString().trim())) { + res = new HashMap<>(); + } else { + res = JSON.parseObject(val.toString(), Map.class); + } + } else { + // 其他类全以字符串处理 + res = val.toString(); + } + + return res; + } + + /** + * Blob转byte[] + */ + private static byte[] blobToBytes(Blob blob) { + try (InputStream is = blob.getBinaryStream()) { + byte[] b = new byte[(int) blob.length()]; + is.read(b); + return b; + } catch (IOException | SQLException e) { + logger.error(e.getMessage()); + return null; + } + } + + /** + * 拼接主键条件 + * + * @param mapping + * @param data + * @return + */ + public static String pkConditaionSql(ESMapping mapping, Map data) { + Set idColumns = new LinkedHashSet<>(); + SchemaItem schemaItem = mapping.getSchemaItem(); + + TableItem mainTable = schemaItem.getMainTable(); + + for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) { + if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null) + || (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) { + idColumns.add(idColumnItem); + } + } + + if (idColumns.isEmpty()) { + throw new RuntimeException("Not found primary key field in main table"); + } + + // 拼接condition + StringBuilder condition = new StringBuilder(" "); + for (ColumnItem idColumn : idColumns) { + Object idVal = data.get(Util.cleanColumn(idColumn.getColumnName())); + if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append("."); + condition.append(idColumn.getColumnName()).append("="); + if (idVal instanceof String) { + condition.append("'").append(idVal).append("' AND "); + } else { + condition.append(idVal).append(" AND "); + } + } + + if (condition.toString().endsWith("AND ")) { + int len2 = condition.length(); + condition.delete(len2 - 4, len2); + } + return condition.toString(); + } + + public static String appendCondition(String sql, String condition) { + return sql + " WHERE " + condition + " "; + } + + /** + * 执行查询sql + */ + public static Object sqlRS(DataSource ds, String sql, Function fun) throws SQLException { + Connection conn = null; + Statement smt = null; + ResultSet rs = null; + try { + conn = ds.getConnection(); + smt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + smt.setFetchSize(Integer.MIN_VALUE); + rs = smt.executeQuery(sql); + + return fun.apply(rs); + } catch (SQLException e) { + logger.error("sqlRs has error, sql: {} ", sql); + throw e; + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error("error to close result set"); + } + } + if (smt != null) { + try { + smt.close(); + } catch (SQLException e) { + logger.error("error to close statement"); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("error to close db connection"); + } + } + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 4950a41c..013b0f0d 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -2,7 +2,10 @@ package com.alibaba.otter.canal.client.adapter.es.support; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -11,6 +14,8 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -101,29 +106,29 @@ public class ESTemplate { } public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESSyncConfig esSyncConfig, Object pkVal, - Map esFieldData) { + Map esFieldData) { ESMapping mapping = esSyncConfig.getEsMapping(); if (mapping.get_id() != null) { bulkRequestBuilder - .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) - .setDoc(esFieldData)); + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setDoc(esFieldData)); } else { SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); for (SearchHit hit : response.getHits()) { // 理论上只有一条 bulkRequestBuilder - .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) - .setDoc(esFieldData)); + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) + .setDoc(esFieldData)); } } } /** * update by query - * + * * @param esSyncConfig * @param queryBuilder * @param esFieldData @@ -215,7 +220,7 @@ public class ESTemplate { /** * 通过主键删除数据 - * + * * @param esSyncConfig * @param pkVal * @return @@ -242,7 +247,7 @@ public class ESTemplate { /** * 批量提交 - * + * * @param bulkRequestBuilder * @return */ @@ -265,7 +270,72 @@ public class ESTemplate { return !response.hasFailures(); } - return true; } + + public Object getDataValue(ESMapping config, Map data, String fieldName) { + String esType = getEsType(config, fieldName); + Object value = data.get(fieldName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + return value; + } + + public Object convertType(ESMapping config, String fieldName, Object val) { + // 从mapping中获取类型来转换 + String esType = getEsType(config, fieldName); + return ESSyncUtil.typeConvert(val, esType); + } + + /** + * es 字段类型本地缓存 + */ + private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + + @SuppressWarnings("unchecked") + public String getEsType(ESMapping config, String fieldName) { + String key = config.get_index() + "-" + config.get_type(); + Map fieldType = esFieldTypes.get(key); + if (fieldType == null) { + ImmutableOpenMap mappings; + try { + mappings = transportClient.admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices() + .get(config.get_index()) + .getMappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + config.get_index()); + } + MappingMetaData mappingMetaData = mappings.get(config.get_type()); + if (mappingMetaData == null) { + throw new IllegalArgumentException("Not found the mapping info of index: " + config.get_index()); + } + + Map fieldTypeTmp = new LinkedHashMap<>(); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map mapping = (Map) sourceMap.get("properties"); + mapping.forEach((k, v) -> { + Map value = (Map) v; + if (value.containsKey("properties")) { + fieldTypeTmp.put(k, "object"); + } else { + fieldTypeTmp.put(k, (String) value.get("type")); + } + }); + fieldType = fieldTypeTmp; + esFieldTypes.put(key, fieldType); + } + + return fieldType.get(fieldName); + } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java new file mode 100644 index 00000000..a801b9e6 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.client.adapter.es.support; + +public class Util { + public static String cleanColumn(String column) { + if (column == null) { + return null; + } + if (column.contains("`")) { + column = column.replaceAll("`", ""); + } + + if (column.contains("'")) { + column = column.replaceAll("'", ""); + } + + return column; + } +} diff --git a/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml b/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml deleted file mode 100644 index d2e38c46..00000000 --- a/client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml +++ /dev/null @@ -1,12 +0,0 @@ -dataSourceKey: defaultDS -esMapping: - _index: mytest_user - _type: _doc - _id: id -# pk: id - sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a - left join role b on a.role_id=b.id - left join (select user_id, group_concat(label,',') as labels from user_label - group by user_id) c on c.user_id=a.id" - commitBatch: 3000 - etlCondition: "where a.c_time>='{0}' or b.c_time>='{0}' or c.c_time>='{0}'" \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml new file mode 100644 index 00000000..63df11eb --- /dev/null +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -0,0 +1,14 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: id +# pk: id +# sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a +# left join role b on a.role_id=b.id +# left join (select user_id, group_concat(label,',') as labels from user_label +# group by user_id) c on c.user_id=a.id" + sql: "select id, name, c_time from user " + commitBatch: 3000 + etlCondition: "where a.c_time>='{0}' or b.c_time>='{0}' or c.c_time>='{0}'" \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java index c2a4761e..81f09937 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -1,6 +1,5 @@ package com.alibaba.otter.canal.client.adapter.es.test; -import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -16,8 +15,8 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; public class ConfigLoadTest { @Before - public void before() throws SQLException { - AdapterConfigs.put("es", "myetst_user.yml"); + public void before() { + AdapterConfigs.put("es", "mytest_user.yml"); // 加载数据源连接池 DatasourceConfig.DATA_SOURCES.put("defaultDS", DataSourceConstant.dataSource); } @@ -26,7 +25,7 @@ public class ConfigLoadTest { public void testLoad() { ESSyncConfigLoader.load(); Map configMap = ESSyncConfigLoader.getEsSyncConfig(); - ESSyncConfig config = configMap.get("myetst_user.yml"); + ESSyncConfig config = configMap.get("mytest_user.yml"); Assert.assertNotNull(config); Assert.assertEquals("defaultDS", config.getDataSourceKey()); ESSyncConfig.ESMapping esMapping = config.getEsMapping(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java new file mode 100644 index 00000000..e75b0beb --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java @@ -0,0 +1,54 @@ +package com.alibaba.otter.canal.client.adapter.es.test; + +import java.util.*; + +import com.alibaba.otter.canal.client.adapter.support.Dml; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +public class SyncTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user.yml"); + DatasourceConfig.DATA_SOURCES.put("defaultDS", DataSourceConstant.dataSource); + + OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); + outerAdapterConfig.setName("es"); + outerAdapterConfig.setHosts("127.0.0.1:9300"); + Map properties = new HashMap<>(); + properties.put("cluster.name", "elasticsearch"); + outerAdapterConfig.setProperties(properties); + + esAdapter = new ESAdapter(); + esAdapter.init(outerAdapterConfig); + + } + + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("c_time", new Date()); + + dml.setData(dataList); + + esAdapter.sync(dml); + } +} diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 52b3da23..2b2d59ff 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -53,7 +53,7 @@ public class HbaseAdapter implements OuterAdapter { hbaseMapping = MappingConfigLoader.load(); mappingConfigCache = new HashMap<>(); for (MappingConfig mappingConfig : hbaseMapping.values()) { - mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseMapping().getDestination()) + mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getHbaseMapping().getDatabase() + "." + mappingConfig.getHbaseMapping().getTable(), mappingConfig); @@ -170,7 +170,7 @@ public class HbaseAdapter implements OuterAdapter { public String getDestination(String task) { MappingConfig config = hbaseMapping.get(task); if (config != null && config.getHbaseMapping() != null) { - return config.getHbaseMapping().getDestination(); + return config.getDestination(); } return null; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 3ff1bb85..c380b65c 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -12,6 +12,8 @@ public class MappingConfig { private String dataSourceKey; // 数据源key + private String destination; // canal实例或MQ的topic + private HbaseMapping hbaseMapping; // hbase映射配置 public String getDataSourceKey() { @@ -22,6 +24,14 @@ public class MappingConfig { this.dataSourceKey = dataSourceKey; } + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + public HbaseMapping getHbaseMapping() { return hbaseMapping; } @@ -143,7 +153,6 @@ public class MappingConfig { public static class HbaseMapping { private Mode mode = Mode.STRING; // hbase默认转换格式 - private String destination; // canal实例或MQ的topic private String database; // 数据库名或schema名 private String table; // 表面名 private String hbaseTable; // hbase表名 @@ -169,14 +178,6 @@ public class MappingConfig { this.mode = mode; } - public String getDestination() { - return destination; - } - - public void setDestination(String destination) { - this.destination = destination; - } - public String getDatabase() { return database; } diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index f31341a3..cbfe6bb1 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -1,7 +1,7 @@ dataSourceKey: defaultDS +destination: example hbaseMapping: mode: PHOENIX #NATIVE #STRING - destination: example database: mytest # 数据库名 table: person2 # 数据库表名 hbaseTable: MYTEST.PERSON2 # HBase表名 diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 26cab04c..3e0e0dc4 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -3,6 +3,7 @@ server: logging: level: com.alibaba.otter.canal.client.adapter.hbase: DEBUG + com.alibaba.otter.canal.client.adapter.es: DEBUG spring: jackson: date-format: yyyy-MM-dd HH:mm:ss @@ -15,30 +16,34 @@ hbase.zookeeper.property.clientPort: 2181 hbase.zookeeper.znode.parent: /hbase canal.conf: -# canalServerHost: 127.0.0.1:11111 + canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 - bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 +# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 flatMessage: true retry: 3 timeout: 20000 batchSize: 50 -# canalInstances: -# - instance: example -# groups: -# - outAdapters: + canalInstances: + - instance: example + groups: + - outAdapters: + - name: es + hosts: 127.0.0.1:9300 + properties: + cluster.name: elasticsearch # - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} # hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} # zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} - mqTopics: - - mqMode: kafka - topic: example - groups: - - groupId: g2 - outAdapters: - - name: logger +# mqTopics: +# - mqMode: kafka +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger # mqTopics: # - mqMode: rocketmq # topic: example @@ -47,11 +52,12 @@ canal.conf: # outAdapters: # - name: logger -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: # - hbase/mytest_person2.yml + - es/mytest_user.yml From dd8243a56d99f5cee72fe6e7065085c36fb7f767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Tue, 30 Oct 2018 23:48:05 +0800 Subject: [PATCH 11/42] fixed issue #1075, support message transactor filter --- .../otter/canal/common/CanalMessageSerializer.java | 8 +++++++- .../alibaba/otter/canal/kafka/MessageSerializer.java | 10 +++++++++- .../otter/canal/rocketmq/CanalRocketMQProducer.java | 3 ++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java b/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java index 665dbf55..fda5dc65 100644 --- a/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java +++ b/server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java @@ -16,7 +16,7 @@ import com.google.protobuf.WireFormat; public class CanalMessageSerializer { @SuppressWarnings("deprecation") - public static byte[] serializer(Message data) { + public static byte[] serializer(Message data, boolean filterTransactionEntry) { try { if (data != null) { if (data.getId() != -1) { @@ -53,8 +53,14 @@ public class CanalMessageSerializer { output.checkNoSpaceLeft(); return body; } else if (!CollectionUtils.isEmpty(data.getEntries())) { + // mq模式只会走到非rowEntry模式 CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); for (CanalEntry.Entry entry : data.getEntries()) { + if (filterTransactionEntry + && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) { + continue; + } + messageBuilder.addMessages(entry.toByteString()); } diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java b/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java index 77caa0a6..39c297d8 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.kafka; import java.util.Map; +import org.apache.commons.lang.BooleanUtils; import org.apache.kafka.common.serialization.Serializer; import com.alibaba.otter.canal.common.CanalMessageSerializer; @@ -15,13 +16,20 @@ import com.alibaba.otter.canal.protocol.Message; */ public class MessageSerializer implements Serializer { + private boolean filterTransactionEntry = false; + + public MessageSerializer(){ + this.filterTransactionEntry = BooleanUtils.toBoolean(System.getProperty("canal.instance.filter.transaction.entry", + "false")); + } + @Override public void configure(Map configs, boolean isKey) { } @Override public byte[] serialize(String topic, Message data) { - return CanalMessageSerializer.serializer(data); + return CanalMessageSerializer.serializer(data, filterTransactionEntry); } @Override diff --git a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java index aaace132..19872490 100644 --- a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java +++ b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java @@ -47,7 +47,8 @@ public class CanalRocketMQProducer implements CanalMQProducer { Callback callback) { if (!mqProperties.getFlatMessage()) { try { - Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data)); + Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data, + mqProperties.isFilterTransactionEntry())); logger.debug("send message:{} to destination:{}, partition: {}", message, destination.getCanalDestination(), From 1d9e2cceddf722728a8614b8204674fc176f41ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 31 Oct 2018 12:51:02 +0800 Subject: [PATCH 12/42] fixed issue #1076 , support rds binlog expired --- .../parse/inbound/mysql/MysqlEventParser.java | 17 +++++++++++- .../mysql/rds/BinlogDownloadQueue.java | 5 ++++ .../mysql/rds/RdsBinlogEventParserProxy.java | 27 +++++++++++++------ .../mysql/rds/RdsLocalBinlogEventParser.java | 10 ++++++- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index df9c952c..45f2559f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -69,6 +69,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // update by yishun.chen,特殊异常处理参数 private int dumpErrorCount = 0; // binlogDump失败异常计数 private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 + private boolean rdsOssMode = false; protected ErosaConnection buildErosaConnection() { return buildMysqlConnection(this.runningInfo); @@ -352,7 +353,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE return logPosition.getPostion(); } - if (masterPosition!=null && StringUtils.isNotEmpty(masterPosition.getGtid())) { + if (masterPosition != null && StringUtils.isNotEmpty(masterPosition.getGtid())) { return masterPosition; } } @@ -493,6 +494,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE dumpErrorCount = 0; return findPosition; } + + Long timestamp = logPosition.getPostion().getTimestamp(); + if (isRdsOssMode() && (timestamp != null && timestamp > 0)) { + // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理 + return null; + } } // 其余情况 logger.warn("prepare to find start position just last position\n {}", @@ -905,4 +912,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE this.dumpErrorCountThreshold = dumpErrorCountThreshold; } + public boolean isRdsOssMode() { + return rdsOssMode; + } + + public void setRdsOssMode(boolean rdsOssMode) { + this.rdsOssMode = rdsOssMode; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java index e132514b..9a5cd271 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java @@ -41,6 +41,7 @@ import org.apache.http.ssl.TrustStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile; /** @@ -103,6 +104,9 @@ public class BinlogDownloadQueue { public BinlogFile tryOne() throws Throwable { BinlogFile binlogFile = binlogList.poll(); + if (binlogFile == null) { + throw new CanalParseException("download binlog is null"); + } download(binlogFile); hostId = binlogFile.getHostInstanceID(); this.currentSize++; @@ -162,6 +166,7 @@ public class BinlogDownloadQueue { this.currentSize = 0; binlogList.clear(); downloadQueue.clear(); + downloadThread = null; } private void download(BinlogFile binlogFile) throws Throwable { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java index 3c38ed0c..f3a220b0 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java @@ -22,14 +22,14 @@ import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; */ public class RdsBinlogEventParserProxy extends MysqlEventParser { - private String rdsOpenApiUrl = "https://rds.aliyuncs.com/"; // openapi地址 - private String accesskey; // 云账号的ak - private String secretkey; // 云账号sk - private String instanceId; // rds实例id - private String directory; // binlog目录 - private int batchFileSize = 4; // 最多下载的binlog文件数量 + private String rdsOpenApiUrl = "https://rds.aliyuncs.com/"; // openapi地址 + private String accesskey; // 云账号的ak + private String secretkey; // 云账号sk + private String instanceId; // rds实例id + private String directory; // binlog目录 + private int batchFileSize = 4; // 最多下载的binlog文件数量 - private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser(); + private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = null; private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() { @Override @@ -43,8 +43,11 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser { @Override public void start() { - if (StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey) + if (rdsLocalBinlogEventParser == null && StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey) && StringUtils.isNotEmpty(instanceId)) { + rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser(); + // rds oss mode + setRdsOssMode(true); final ParserExceptionHandler targetHandler = this.getParserExceptionHandler(); if (directory == null) { directory = System.getProperty("java.io.tmpdir", "/tmp") + "/" + destination; @@ -119,10 +122,18 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser { long serverId = rdsBinlogEventParserProxy.getServerId(); rdsLocalBinlogEventParser.setServerId(serverId); rdsBinlogEventParserProxy.stop(); + } catch (Throwable e) { + logger.info("handle exception failed", e); + } + + try { logger.info("start rds mysql binlog parser!"); rdsLocalBinlogEventParser.start(); } catch (Throwable e) { logger.info("handle exception failed", e); + rdsLocalBinlogEventParser.stop(); + RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this; + rdsBinlogEventParserProxy.start();// 继续重试 } } }); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java index aac14d99..2e0960fe 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java @@ -57,7 +57,11 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements if (entryPosition == null) { throw new PositionNotFoundException("position not found!"); } - long startTimeInMill = entryPosition.getTimestamp(); + Long startTimeInMill = entryPosition.getTimestamp(); + if (startTimeInMill == null || startTimeInMill <= 0) { + throw new PositionNotFoundException("position timestamp is empty!"); + } + startTime = startTimeInMill; List binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, accesskey, @@ -65,6 +69,10 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements instanceId, new Date(startTime), new Date(endTime)); + if (binlogFiles.isEmpty()) { + throw new CanalParseException("start timestamp : " + startTimeInMill + " binlog files is empty"); + } + binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchFileSize, directory); binlogDownloadQueue.silenceDownload(); needWait = true; From 2a2977eca7b75b4af207bc06982ab578fbf818d8 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 31 Oct 2018 14:00:06 +0800 Subject: [PATCH 13/42] =?UTF-8?q?=E4=B8=BB=E8=A1=A8=E5=B8=A6=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E6=96=B0=E5=A2=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 8 + .../adapter/es/service/ESSyncService.java | 127 ++++++------- .../client/adapter/es/support/ESSyncUtil.java | 6 +- .../client/adapter/es/support/ESTemplate.java | 169 +++++++++++++++--- .../src/main/resources/es/mytest_user.yml | 3 +- .../adapter/es/test/ConfigLoadTest.java | 2 +- .../client/adapter/es/test/SyncTest.java | 54 ------ ...aSourceConstant.java => TestConstant.java} | 21 ++- .../adapter/es/test/sync/UserSyncTest.java | 97 ++++++++++ .../src/test/resources/logback-test.xml | 13 ++ .../loader/AbstractCanalAdapterWorker.java | 2 + 11 files changed, 337 insertions(+), 165 deletions(-) delete mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java rename client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/{DataSourceConstant.java => TestConstant.java} (57%) create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java create mode 100644 client-adapter/elasticsearch/src/test/resources/logback-test.xml diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index 983dedc9..e5cc318b 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -35,6 +35,14 @@ public class ESAdapter implements OuterAdapter { private ESSyncService esSyncService; + public TransportClient getTransportClient() { + return transportClient; + } + + public ESSyncService getEsSyncService() { + return esSyncService; + } + @Override public void init(OuterAdapterConfig configuration) { try { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index d028f6ba..35d973d0 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -1,6 +1,6 @@ package com.alibaba.otter.canal.client.adapter.es.service; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -14,10 +14,8 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; -import com.alibaba.otter.canal.client.adapter.es.support.Util; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; @@ -32,32 +30,38 @@ public class ESSyncService { } public void sync(Dml dml) { + if (logger.isDebugEnabled()) { + logger.debug("DML: {}", JSON.toJSONString(dml)); + } long begin = System.currentTimeMillis(); String database = dml.getDatabase(); String table = dml.getTable(); List esSyncConfigs = ESSyncConfigLoader.getDbTableEsSyncConfig().get(database + "-" + table); if (esSyncConfigs != null) { - if (logger.isDebugEnabled()) { - logger.debug("Destination: {}, database:{}, table:{}, type:{}, effect index count: {}", + if (logger.isTraceEnabled()) { + logger.trace("Destination: {}, database:{}, table:{}, type:{}, effect index count: {}", dml.getDestination(), dml.getDatabase(), dml.getTable(), dml.getType(), esSyncConfigs.size()); - logger.debug(JSON.toJSONString(dml)); } for (ESSyncConfig config : esSyncConfigs) { - logger.info("Prepared to sync index: {}, destination: {}", - config.getEsMapping().get_index(), - dml.getDestination()); + if (logger.isTraceEnabled()) { + logger.trace("Prepared to sync index: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } this.sync(config, dml); - logger.info("Sync completed: {}, destination: {}", - config.getEsMapping().get_index(), - dml.getDestination()); + if (logger.isTraceEnabled()) { + logger.trace("Sync completed: {}, destination: {}", + config.getEsMapping().get_index(), + dml.getDestination()); + } } - if (logger.isDebugEnabled()) { - logger.debug("Sync elapsed time: {}, effect index count：{}, destination: {}", + if (logger.isTraceEnabled()) { + logger.trace("Sync elapsed time: {}, effect index count：{}, destination: {}", (System.currentTimeMillis() - begin), esSyncConfigs.size(), dml.getDestination()); @@ -78,8 +82,8 @@ public class ESSyncService { // delete(config, dml); } - if (logger.isDebugEnabled()) { - logger.debug("Sync elapsed time: {},destination: {}, es index: {}", + if (logger.isTraceEnabled()) { + logger.trace("Sync elapsed time: {},destination: {}, es index: {}", (System.currentTimeMillis() - begin), dml.getDestination(), config.getEsMapping().get_index()); @@ -104,39 +108,11 @@ public class ESSyncService { // ------是否单表 & 所有字段都为简单字段------ if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { - Map esFieldData = new HashMap<>(); + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData); - // ------通过dml对象插入ES------ - // 获取所有字段 - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - // 如果是主键字段则不插入 - if (fieldItem.getFieldName().equals(mapping.get_id())) { - continue; - } - Object value = esTemplate.getDataValue(mapping, - data, - Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName())); - String fieldName = Util.cleanColumn(fieldItem.getFieldName()); - if (!mapping.getSkips().contains(fieldName)) { - esFieldData.put(fieldName, esTemplate.convertType(mapping, fieldName, value)); - } - } - - // 取出id值 - Object idVal; - if (mapping.get_id() != null) { - idVal = esTemplate.getDataValue(mapping, - data, - Util.cleanColumn( - schemaItem.getSelectFields().get(mapping.get_id()).getColumn().getColumnName())); - } else { - idVal = esTemplate.getDataValue(mapping, - data, - Util.cleanColumn( - schemaItem.getSelectFields().get(mapping.getPk()).getColumn().getColumnName())); - } - if (logger.isDebugEnabled()) { - logger.debug("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}", + if (logger.isTraceEnabled()) { + logger.trace("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), @@ -150,35 +126,44 @@ public class ESSyncService { mapping.get_index(), idVal); } - return; // 单表插入完成返回 + continue; // 单表插入完成 } - // ------是否主表------ + // ------是否主表 查询sql来插入------ if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { String sql = mapping.getSql(); - String condition = ESSyncUtil.pkConditaionSql(mapping, data); + String condition = ESSyncUtil.pkConditionSql(mapping, data); sql = ESSyncUtil.appendCondition(sql, condition); DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); -// ESSyncUtil.sqlRS(ds, sql, rs -> { -// try { -// while (rs.next()) { -// Map esFieldData = new HashMap<>(); -// -// Object idVal = ESSyncUtil.persistEsByRs(transportClient, config, rs, mainTable, esFieldData); -// -// if (logger.isDebugEnabled()) { -// logger.debug("主表insert, 通过执行sql查询相应记录，并insert到Es, id: {}, esFieldValue: {}", idVal, esFieldData); -// } -// boolean result = ESSyncUtil.persist(transportClient, config, idVal, esFieldData, OpType.INSERT); -// if (!result) { -// logger.error("主表insert，通过执行sql查询响应记录，并insert到es, es插入失败，table: {}, index: {}", dml.getTable(), config.getEsSyn().getIndex()); -// } -// } -// } catch (Exception e) { -// throw new RuntimeException(e); -// } -// return 0; -// }); + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.insert(config, idVal, esFieldData); + if (!result) { + logger.error( + "Single table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); } } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index 5cd436f5..b86b13e6 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -208,7 +208,7 @@ public class ESSyncUtil { * @param data * @return */ - public static String pkConditaionSql(ESMapping mapping, Map data) { + public static String pkConditionSql(ESMapping mapping, Map data) { Set idColumns = new LinkedHashSet<>(); SchemaItem schemaItem = mapping.getSchemaItem(); @@ -252,7 +252,7 @@ public class ESSyncUtil { /** * 执行查询sql */ - public static Object sqlRS(DataSource ds, String sql, Function fun) throws SQLException { + public static Object sqlRS(DataSource ds, String sql, Function fun) { Connection conn = null; Statement smt = null; ResultSet rs = null; @@ -265,7 +265,7 @@ public class ESSyncUtil { return fun.apply(rs); } catch (SQLException e) { logger.error("sqlRs has error, sql: {} ", sql); - throw e; + throw new RuntimeException(e); } finally { if (rs != null) { try { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 013b0f0d..89d8e4ed 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -1,5 +1,7 @@ package com.alibaba.otter.canal.client.adapter.es.support; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -31,6 +33,8 @@ import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; public class ESTemplate { @@ -177,8 +181,8 @@ public class ESTemplate { } }); String scriptLine = sb.toString(); - if (logger.isDebugEnabled()) { - logger.debug(scriptLine); + if (logger.isTraceEnabled()) { + logger.trace(scriptLine); } UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); @@ -188,8 +192,8 @@ public class ESTemplate { .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); BulkByScrollResponse response = updateByQuery.get(); - if (logger.isDebugEnabled()) { - logger.debug("updateByQuery response: {}", response.getStatus()); + if (logger.isTraceEnabled()) { + logger.trace("updateByQuery response: {}", response.getStatus()); } if (!CollectionUtils.isEmpty(response.getSearchFailures())) { logger.error("script update_for_search has search error: " + response.getBulkFailures()); @@ -273,21 +277,122 @@ public class ESTemplate { return true; } - public Object getDataValue(ESMapping config, Map data, String fieldName) { - String esType = getEsType(config, fieldName); - Object value = data.get(fieldName); - if (value instanceof Byte) { - if ("boolean".equals(esType)) { - value = ((Byte) value).intValue() != 0; + // public Object getRSDataValue(ESMapping mapping, ResultSet rs, String + // fieldName) throws SQLException { + // String esType = getEsType(mapping, fieldName); + // Object value = rs.getObject(fieldName); + // if (value instanceof Boolean) { + // // 判断es类型 + // if (!"boolean".equals(esType)) { + // value = rs.getByte(fieldName); + // } + // } + // return value; + // } + // + // public Object getDataValue(ESMapping mapping, Map data, + // String fieldName) { + // String esType = getEsType(mapping, fieldName); + // Object value = data.get(fieldName); + // if (value instanceof Byte) { + // if ("boolean".equals(esType)) { + // value = ((Byte) value).intValue() != 0; + // } + // } + // return value; + // } + + // public Object convertType(ESMapping mapping, String fieldName, Object val) { + // // 从mapping中获取类型来转换 + // String esType = getEsType(mapping, fieldName); + // return ESSyncUtil.typeConvert(val, esType); + // } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String fieldName = Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(fieldName); + if (value instanceof Boolean) { + // 判断es类型 + if (!"boolean".equals(esType)) { + value = resultSet.getByte(fieldName); + } + } + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = ESSyncUtil.typeConvert(value, esType); + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) && !mapping.getSkips().contains(fieldName)) { + esFieldData.put(fieldName, ESSyncUtil.typeConvert(value, esType)); } } - return value; + return resultIdVal; } - public Object convertType(ESMapping config, String fieldName, Object val) { - // 从mapping中获取类型来转换 - String esType = getEsType(config, fieldName); - return ESSyncUtil.typeConvert(val, esType); + // public Object getIdValFromDmlData(ESMapping mapping, Map + // dmlData) { + // SchemaItem schemaItem = mapping.getSchemaItem(); + // String idFieldName = mapping.get_id() == null ? mapping.getPk() : + // mapping.get_id(); + // + // for (SchemaItem.FieldItem fieldItem : schemaItem.getSelectFields().values()) + // { + // if (fieldItem.getFieldName().equals(idFieldName)) { + // String fieldName = + // Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); + // String esType = getEsType(mapping, fieldName); + // Object value = dmlData.get(fieldName); + // if (value instanceof Byte) { + // if ("boolean".equals(esType)) { + // value = ((Byte) value).intValue() != 0; + // } + // } + // + // return ESSyncUtil.typeConvert(value, esType); + // } + // } + // return null; + // } + + /** + * 将dml的data转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String fieldName = Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(fieldName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = ESSyncUtil.typeConvert(value, esType); + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) && !mapping.getSkips().contains(fieldName)) { + esFieldData.put(fieldName, ESSyncUtil.typeConvert(value, esType)); + } + } + return resultIdVal; } /** @@ -295,9 +400,16 @@ public class ESTemplate { */ private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + /** + * 获取es mapping中的属性类型 + * + * @param mapping mapping配置 + * @param fieldName 属性名 + * @return 类型 + */ @SuppressWarnings("unchecked") - public String getEsType(ESMapping config, String fieldName) { - String key = config.get_index() + "-" + config.get_type(); + private String getEsType(ESMapping mapping, String fieldName) { + String key = mapping.get_index() + "-" + mapping.get_type(); Map fieldType = esFieldTypes.get(key); if (fieldType == null) { ImmutableOpenMap mappings; @@ -310,29 +422,28 @@ public class ESTemplate { .getState() .getMetaData() .getIndices() - .get(config.get_index()) + .get(mapping.get_index()) .getMappings(); } catch (NullPointerException e) { - throw new IllegalArgumentException("Not found the mapping info of index: " + config.get_index()); + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); } - MappingMetaData mappingMetaData = mappings.get(config.get_type()); + MappingMetaData mappingMetaData = mappings.get(mapping.get_type()); if (mappingMetaData == null) { - throw new IllegalArgumentException("Not found the mapping info of index: " + config.get_index()); + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); } - Map fieldTypeTmp = new LinkedHashMap<>(); + fieldType = new LinkedHashMap<>(); Map sourceMap = mappingMetaData.getSourceAsMap(); - Map mapping = (Map) sourceMap.get("properties"); - mapping.forEach((k, v) -> { - Map value = (Map) v; + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); if (value.containsKey("properties")) { - fieldTypeTmp.put(k, "object"); + fieldType.put(entry.getKey(), "object"); } else { - fieldTypeTmp.put(k, (String) value.get("type")); + fieldType.put(entry.getKey(), (String) value.get("type")); } - }); - fieldType = fieldTypeTmp; + } esFieldTypes.put(key, fieldType); } diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 63df11eb..6b2dbec7 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -9,6 +9,7 @@ esMapping: # left join role b on a.role_id=b.id # left join (select user_id, group_concat(label,',') as labels from user_label # group by user_id) c on c.user_id=a.id" - sql: "select id, name, c_time from user " +# sql: "select id, name, c_time from user " + sql: "select id, concat(name, '_') as name, c_time from user " commitBatch: 3000 etlCondition: "where a.c_time>='{0}' or b.c_time>='{0}' or c.c_time>='{0}'" \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java index 81f09937..83f0cfb9 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -18,7 +18,7 @@ public class ConfigLoadTest { public void before() { AdapterConfigs.put("es", "mytest_user.yml"); // 加载数据源连接池 - DatasourceConfig.DATA_SOURCES.put("defaultDS", DataSourceConstant.dataSource); + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); } @Test diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java deleted file mode 100644 index e75b0beb..00000000 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SyncTest.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.es.test; - -import java.util.*; - -import com.alibaba.otter.canal.client.adapter.support.Dml; -import org.junit.Before; -import org.junit.Test; - -import com.alibaba.otter.canal.client.adapter.es.ESAdapter; -import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; - -public class SyncTest { - - private ESAdapter esAdapter; - - @Before - public void init() { - AdapterConfigs.put("es", "mytest_user.yml"); - DatasourceConfig.DATA_SOURCES.put("defaultDS", DataSourceConstant.dataSource); - - OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); - outerAdapterConfig.setName("es"); - outerAdapterConfig.setHosts("127.0.0.1:9300"); - Map properties = new HashMap<>(); - properties.put("cluster.name", "elasticsearch"); - outerAdapterConfig.setProperties(properties); - - esAdapter = new ESAdapter(); - esAdapter.init(outerAdapterConfig); - - } - - @Test - public void insertTest01() { - Dml dml = new Dml(); - dml.setDestination("example"); - dml.setTs(new Date().getTime()); - dml.setType("INSERT"); - dml.setDatabase("mytest"); - dml.setTable("user"); - List> dataList = new ArrayList<>(); - Map data = new LinkedHashMap<>(); - dataList.add(data); - data.put("id", 1L); - data.put("name", "Eric"); - data.put("c_time", new Date()); - - dml.setData(dataList); - - esAdapter.sync(dml); - } -} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java similarity index 57% rename from client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java rename to client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java index c29953cd..46b0f7a9 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java @@ -4,18 +4,26 @@ import java.sql.SQLException; import com.alibaba.druid.pool.DruidDataSource; -public class DataSourceConstant { +public class TestConstant { + + public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"; + public final static String jdbcUser = "root"; + public final static String jdbcPassword = "121212"; + + public final static String esHosts = "127.0.0.1:9300"; + public final static String clusterNmae = "elasticsearch"; + + public static DruidDataSource dataSource; - static DruidDataSource dataSource; static { dataSource = new DruidDataSource(); dataSource.setDriverClassName("com.mysql.jdbc.Driver"); - dataSource.setUrl("jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"); - dataSource.setUsername("root"); - dataSource.setPassword("121212"); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUser); + dataSource.setPassword(jdbcPassword); dataSource.setInitialSize(1); dataSource.setMinIdle(1); - dataSource.setMaxActive(3); + dataSource.setMaxActive(1); dataSource.setMaxWait(60000); dataSource.setTimeBetweenEvictionRunsMillis(60000); dataSource.setMinEvictableIdleTimeMillis(300000); @@ -28,4 +36,5 @@ public class DataSourceConstant { e.printStackTrace(); } } + } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java new file mode 100644 index 00000000..2f152eb8 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java @@ -0,0 +1,97 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.test.TestConstant; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +public class UserSyncTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user.yml"); + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + + OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); + outerAdapterConfig.setName("es"); + outerAdapterConfig.setHosts(TestConstant.esHosts); + Map properties = new HashMap<>(); + properties.put("cluster.name", TestConstant.clusterNmae); + outerAdapterConfig.setProperties(properties); + + esAdapter = new ESAdapter(); + esAdapter.init(outerAdapterConfig); + + } + + /** + * 单表插入 + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("c_time", new Date()); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric", response.getSource().get("name")); + } + + /** + * 主表带函数插入, 数据库里内容必须和单测一致 + */ + @Test + public void insertTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("c_time", new Date()); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric_", response.getSource().get("name")); + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/resources/logback-test.xml b/client-adapter/elasticsearch/src/test/resources/logback-test.xml new file mode 100644 index 00000000..91f0de80 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index 9e7adeca..e5dce8ae 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -150,6 +150,8 @@ public abstract class AbstractCanalAdapterWorker { } catch (Throwable e) { if (i == retry - 1) { connector.ack(); + } else { + connector.rollback(); } logger.error(e.getMessage(), e); From ff3ccde62a3bc8dac5ae74872f578d9bd0186cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 31 Oct 2018 14:25:54 +0800 Subject: [PATCH 14/42] fixed rds binlog dump --- .../parse/inbound/AbstractEventParser.java | 6 +++++- .../parse/inbound/mysql/MysqlEventParser.java | 19 +++++++++++-------- .../inbound/mysql/dbsync/LogEventConvert.java | 4 ++-- .../mysql/rds/BinlogDownloadQueue.java | 11 +++++++++-- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index dbb40287..8902c05e 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -353,7 +353,11 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle eventSink.interrupt(); if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) { - multiStageCoprocessor.stop(); + try { + multiStageCoprocessor.stop(); + } catch (Throwable t) { + logger.debug("multi processor rejected:", t); + } } try { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 45f2559f..e8ca45e3 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -752,14 +752,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE logPosition.setPostion(entryPosition); } - if (entry == null) { - return true; - } - - String logfilename = entry.getHeader().getLogfileName(); - Long logfileoffset = entry.getHeader().getLogfileOffset(); - Long logposTimestamp = entry.getHeader().getExecuteTime(); - Long serverId = entry.getHeader().getServerId(); + // 直接用event的位点来处理,解决一个binlog文件里没有任何事件导致死循环无法退出的问题 + String logfilename = event.getHeader().getLogFileName(); + // 记录的是binlog end offest, + // 因为与其对比的offest是show master status里的end offest + Long logfileoffset = event.getHeader().getLogPos(); + Long logposTimestamp = event.getHeader().getWhen() * 1000; + Long serverId = event.getHeader().getServerId(); // 如果最小的一条记录都不满足条件，可直接退出 if (logposTimestamp >= startTimestamp) { @@ -771,6 +770,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE return false; } + if (entry == null) { + return true; + } + // 记录一下上一个事务结束的位置，即下一个事务的position // position = current + // data.length，代表该事务的下一条offest，避免多余的事务重复 diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index ec71465b..0b6ab412 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -567,8 +567,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } private EntryPosition createPosition(LogHeader logHeader) { - return new EntryPosition(logHeader.getLogFileName(), - logHeader.getLogPos(), + return new EntryPosition(logHeader.getLogFileName(), logHeader.getLogPos() - logHeader.getEventLen(), // startPos logHeader.getWhen() * 1000L, logHeader.getServerId()); // 记录到秒 } @@ -818,6 +817,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar Header.Builder headerBuilder = Header.newBuilder(); headerBuilder.setVersion(version); headerBuilder.setLogfileName(logHeader.getLogFileName()); + // 记录的是该binlog的start offest headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen()); headerBuilder.setServerId(logHeader.getServerId()); headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java index 9a5cd271..b90b89d5 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java @@ -135,7 +135,7 @@ public class BinlogDownloadQueue { if (StringUtils.isNotEmpty(needCompareName) && StringUtils.endsWith(needCompareName, "tar")) { needCompareName = needCompareName.substring(0, needCompareName.indexOf(".")); } - return fileName.equalsIgnoreCase(needCompareName) && binlogList.isEmpty(); + return (needCompareName == null || fileName.equalsIgnoreCase(needCompareName)) && binlogList.isEmpty(); } public void prepare() throws InterruptedException { @@ -166,7 +166,14 @@ public class BinlogDownloadQueue { this.currentSize = 0; binlogList.clear(); downloadQueue.clear(); - downloadThread = null; + try { + downloadThread.interrupt(); + downloadThread.join();// 等待其结束 + } catch (InterruptedException e) { + // ignore + } finally { + downloadThread = null; + } } private void download(BinlogFile binlogFile) throws Throwable { From a4856b9a739bcbef6b7550e6ec0d7158f66b91f3 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 31 Oct 2018 16:55:48 +0800 Subject: [PATCH 15/42] modify --- .../client/adapter/es/config/SchemaItem.java | 102 ++++++++++++++---- .../adapter/es/service/ESSyncService.java | 64 +++++++++++ .../client/adapter/es/support/ESSyncUtil.java | 13 +-- .../client/adapter/es/support/ESTemplate.java | 64 ++++++----- .../src/main/resources/es/mytest_user.yml | 4 +- .../client/adapter/es/test/SqlParseTest.java | 8 +- .../adapter/es/test/sync/UserSyncTest.java | 19 ++++ .../src/main/resources/application.yml | 34 +++--- 8 files changed, 227 insertions(+), 81 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java index aa77f1ed..ccd58773 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; + import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; public class SchemaItem { @@ -22,7 +23,8 @@ public class SchemaItem { this.isAllFieldsSimple(); aliasTableItems.values().forEach(tableItem -> { tableItem.getRelationTableFields(); - tableItem.getRelationSelectFields(); + tableItem.getRelationKeyFieldItems(); + tableItem.getRelationSelectFieldItems(); }); } @@ -131,20 +133,21 @@ public class SchemaItem { public static class TableItem { - private SchemaItem schemaItem; + private SchemaItem schemaItem; - private String schema; - private String tableName; - private String alias; - private String subQuerySql; - private List subQueryFields = new ArrayList<>(); - private List relationFields = new ArrayList<>(); + private String schema; + private String tableName; + private String alias; + private String subQuerySql; + private List subQueryFields = new ArrayList<>(); + private List relationFields = new ArrayList<>(); - private boolean main; - private boolean subQuery; + private boolean main; + private boolean subQuery; - private volatile List relationTableFields; // 当前表关联条件字段 - private volatile List relationSelectFieldItem; // 关联条件字段在select中的对应字段 + private volatile Map> relationTableFields; // 当前表关联条件字段 + private volatile List relationKeyFieldItems; // 关联条件字段在select中的对应字段 + private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 public TableItem(SchemaItem schemaItem){ this.schemaItem = schemaItem; @@ -222,18 +225,40 @@ public class SchemaItem { this.relationFields = relationFields; } - public List getRelationTableFields() { + public Map> getRelationTableFields() { if (relationTableFields == null) { synchronized (SchemaItem.class) { if (relationTableFields == null) { - relationTableFields = new ArrayList<>(); + relationTableFields = new LinkedHashMap<>(); + getRelationFields().forEach(relationFieldsPair -> { FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + FieldItem currentTableRelField = null; if (getAlias().equals(leftFieldItem.getOwner())) { - relationTableFields.add(leftFieldItem); + // relationTableFields.add(leftFieldItem); + currentTableRelField = leftFieldItem; } else if (getAlias().equals(rightFieldItem.getOwner())) { - relationTableFields.add(rightFieldItem); + // relationTableFields.add(rightFieldItem); + currentTableRelField = rightFieldItem; + } + + if (currentTableRelField != null) { + List selectFieldItem = getSchemaItem().getColumnFields() + .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + selectFieldItem = getSchemaItem().getColumnFields() + .get(rightFieldItem.getOwner() + "." + + rightFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + throw new UnsupportedOperationException( + "Relation condition column must in select columns."); + } + } } }); } @@ -242,23 +267,23 @@ public class SchemaItem { return relationTableFields; } - public List getRelationSelectFields() { - if (relationSelectFieldItem == null) { + public List getRelationKeyFieldItems() { + if (relationKeyFieldItems == null) { synchronized (SchemaItem.class) { - if (relationSelectFieldItem == null) { - relationSelectFieldItem = new ArrayList<>(); + if (relationKeyFieldItems == null) { + relationKeyFieldItems = new ArrayList<>(); getRelationFields().forEach(relationFieldsPair -> { FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); List selectFieldItem = getSchemaItem().getColumnFields() .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - relationSelectFieldItem.addAll(selectFieldItem); + relationKeyFieldItems.addAll(selectFieldItem); } else { FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); selectFieldItem = getSchemaItem().getColumnFields() .get(rightFieldItem.getOwner() + "." + rightFieldItem.getColumn().getColumnName()); if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - relationSelectFieldItem.addAll(selectFieldItem); + relationKeyFieldItems.addAll(selectFieldItem); } else { throw new UnsupportedOperationException( "Relation condition column must in select columns."); @@ -268,7 +293,23 @@ public class SchemaItem { } } } - return relationSelectFieldItem; + return relationKeyFieldItems; + } + + public List getRelationSelectFieldItems() { + if (relationSelectFieldItems == null) { + synchronized (SchemaItem.class) { + if (relationSelectFieldItems == null) { + relationSelectFieldItems = new ArrayList<>(); + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getOwners().contains(getAlias())) { + relationSelectFieldItems.add(fieldItem); + } + } + } + } + } + return relationSelectFieldItems; } } @@ -367,6 +408,21 @@ public class SchemaItem { return null; } } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldItem fieldItem = (FieldItem) o; + + return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null; + } + + @Override + public int hashCode() { + return fieldName != null ? fieldName.hashCode() : 0; + } } public static class ColumnItem { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 35d973d0..9eb69c12 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -6,6 +6,8 @@ import java.util.Map; import javax.sql.DataSource; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +16,8 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; @@ -165,6 +169,66 @@ public class ESSyncService { return 0; }); } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + // ------关联条件出现在主表查询条件------ + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate + .getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Map.Entry> entry : tableItem.getRelationTableFields() + .entrySet()) { + Object value = esTemplate + .getValFromData(mapping, data, entry.getKey().getColumn().getColumnName()); + for (FieldItem fieldItem : entry.getValue()) { + queryBuilder.must(QueryBuilders.termsQuery(fieldItem.getFieldName(), value)); + } + } + + // + // for (FieldItem fieldItem : tableItem.getRelationTableFields()) { + // Object value = esTemplate + // .getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); + // + // } + + // if (logger.isDebugEnabled()) { + // logger.debug("从表insert, 且均为简单字段，对es进行update_by_query, queryBuilder: {}, + // esFieldData:{}", queryBuilder.toString(), esFieldData); + // } + boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + // if (!result) { + // logger.error("从表insert，均为简单字段，直接对es进行update_by_query, es更新存在错误, table: {}, + // index: {}, dml: {}", dml.getTable(), config.getEsSyn().getIndex(), dml); + // } + } + // TODO + } else { + // TODO 查询总sql + } + } } } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index b86b13e6..f09641d7 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.es.support; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.*; import java.util.Date; @@ -148,14 +149,10 @@ public class ESSyncUtil { Base64 base64 = new Base64(); res = base64.encodeAsString(b); } else if (val instanceof String) { - try { - // 对应canal中的单字节编码 - byte[] b = ((String) val).getBytes("ISO-8859-1"); - Base64 base64 = new Base64(); - res = base64.encodeAsString(b); - } catch (UnsupportedEncodingException e) { - logger.error(e.getMessage()); - } + // 对应canal中的单字节编码 + byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1); + Base64 base64 = new Base64(); + res = base64.encodeAsString(b); } } else if ("geo_point".equals(esType)) { if (!(val instanceof String)) { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 89d8e4ed..9908c0d8 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -133,24 +133,21 @@ public class ESTemplate { /** * update by query * - * @param esSyncConfig + * @param mapping * @param queryBuilder * @param esFieldData * @return */ - private boolean updateByQuery(ESSyncConfig esSyncConfig, QueryBuilder queryBuilder, - Map esFieldData) { - return updateByQuery(esSyncConfig, queryBuilder, esFieldData, 0); + public boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData) { + return updateByQuery(mapping, queryBuilder, esFieldData, 1); } - private boolean updateByQuery(ESSyncConfig esSyncConfig, QueryBuilder queryBuilder, Map esFieldData, - int counter) { + public boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, + int counter) { if (CollectionUtils.isEmpty(esFieldData)) { return true; } - ESMapping mapping = esSyncConfig.getEsMapping(); - StringBuilder sb = new StringBuilder(); esFieldData.forEach((key, value) -> { if (value instanceof Map) { @@ -216,7 +213,7 @@ public class ESTemplate { } catch (InterruptedException e) { // ignore } - return updateByQuery(esSyncConfig, queryBuilder, esFieldData, ++counter); + return updateByQuery(mapping, queryBuilder, esFieldData, ++counter); } return true; @@ -308,6 +305,18 @@ public class ESTemplate { // return ESSyncUtil.typeConvert(val, esType); // } + public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName) throws SQLException { + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(fieldName); + if (value instanceof Boolean) { + if (!"boolean".equals(esType)) { + value = resultSet.getByte(fieldName); + } + } + return value; + } + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map esFieldData) throws SQLException { SchemaItem schemaItem = mapping.getSchemaItem(); @@ -315,22 +324,14 @@ public class ESTemplate { Object resultIdVal = null; for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { String fieldName = Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); - String esType = getEsType(mapping, fieldName); - - Object value = resultSet.getObject(fieldName); - if (value instanceof Boolean) { - // 判断es类型 - if (!"boolean".equals(esType)) { - value = resultSet.getByte(fieldName); - } - } + Object value = getValFromRS(mapping, resultSet, fieldName); if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = ESSyncUtil.typeConvert(value, esType); + resultIdVal = value; } if (!fieldItem.getFieldName().equals(mapping.get_id()) && !mapping.getSkips().contains(fieldName)) { - esFieldData.put(fieldName, ESSyncUtil.typeConvert(value, esType)); + esFieldData.put(fieldName, value); } } return resultIdVal; @@ -361,6 +362,17 @@ public class ESTemplate { // return null; // } + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName) { + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(fieldName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + return ESSyncUtil.typeConvert(value, esType); + } + /** * 将dml的data转换为es的data * @@ -376,20 +388,14 @@ public class ESTemplate { Object resultIdVal = null; for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { String fieldName = Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); - String esType = getEsType(mapping, fieldName); - Object value = dmlData.get(fieldName); - if (value instanceof Byte) { - if ("boolean".equals(esType)) { - value = ((Byte) value).intValue() != 0; - } - } + Object value = getValFromData(mapping, dmlData, fieldName); if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = ESSyncUtil.typeConvert(value, esType); + resultIdVal = value; } if (!fieldItem.getFieldName().equals(mapping.get_id()) && !mapping.getSkips().contains(fieldName)) { - esFieldData.put(fieldName, ESSyncUtil.typeConvert(value, esType)); + esFieldData.put(fieldName, value); } } return resultIdVal; diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 6b2dbec7..7a407f71 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -10,6 +10,8 @@ esMapping: # left join (select user_id, group_concat(label,',') as labels from user_label # group by user_id) c on c.user_id=a.id" # sql: "select id, name, c_time from user " - sql: "select id, concat(name, '_') as name, c_time from user " +# sql: "select id, concat(name, '_') as name, c_time from user " + sql: "select a.id, a.name, a.role_id, b.role_name, a.c_time from user a + left join role b on b.id=a.role_id" commitBatch: 3000 etlCondition: "where a.c_time>='{0}' or b.c_time>='{0}' or c.c_time>='{0}'" \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java index 6d0644d2..f31e6a89 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java @@ -1,6 +1,7 @@ package com.alibaba.otter.canal.client.adapter.es.test; import java.util.List; +import java.util.Map; import org.junit.Assert; import org.junit.Test; @@ -33,11 +34,12 @@ public class SqlParseTest { fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName())); // 获取当前表关联条件字段 - List relationTableFields = tableItem.getRelationTableFields(); - relationTableFields.forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); + Map> relationTableFields = tableItem.getRelationTableFields(); + relationTableFields.keySet() + .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); // 获取关联字段在select中的对应字段 - List relationSelectFieldItem = tableItem.getRelationSelectFields(); + List relationSelectFieldItem = tableItem.getRelationKeyFieldItems(); relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java index 2f152eb8..9fc9c211 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java @@ -89,6 +89,25 @@ public class UserSyncTest { Assert.assertEquals("Eric_", response.getSource().get("name")); } + @Test + public void insertTest03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 3e0e0dc4..a7d512c4 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -16,34 +16,34 @@ hbase.zookeeper.property.clientPort: 2181 hbase.zookeeper.znode.parent: /hbase canal.conf: - canalServerHost: 127.0.0.1:11111 +# canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 -# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 + bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 flatMessage: true retry: 3 timeout: 20000 batchSize: 50 - canalInstances: - - instance: example - groups: - - outAdapters: - - name: es - hosts: 127.0.0.1:9300 - properties: - cluster.name: elasticsearch +# canalInstances: +# - instance: example +# groups: +# - outAdapters: +# - name: es +# hosts: 127.0.0.1:9300 +# properties: +# cluster.name: elasticsearch # - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} # hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} # zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} -# mqTopics: -# - mqMode: kafka -# topic: example -# groups: -# - groupId: g2 -# outAdapters: -# - name: logger + mqTopics: + - mqMode: kafka + topic: example + groups: + - groupId: g2 + outAdapters: + - name: logger # mqTopics: # - mqMode: rocketmq # topic: example From ed1feb9502e19bfc854b2d82a32253ab469b1449 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 31 Oct 2018 16:58:50 +0800 Subject: [PATCH 16/42] =?UTF-8?q?application.yml=E4=B8=8D=E6=89=93?= =?UTF-8?q?=E5=8C=85=E8=BF=9Bjar,=20=E8=A7=A3=E5=86=B3jar=E4=B8=AD?= =?UTF-8?q?=E7=9A=84application.yml=E5=92=8Cconfig=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E4=BC=9A=E5=90=88=E5=B9=B6bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/launcher/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 1a79ed21..708e1471 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -166,6 +166,14 @@ + + + src/main/resources + + application.yml + + + From 14e094b52210050b4d1e619c268550c622e34f53 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 31 Oct 2018 18:47:16 +0800 Subject: [PATCH 17/42] modify --- .../client/adapter/es/config/SchemaItem.java | 65 ++++++++++--------- .../adapter/es/service/ESSyncService.java | 41 +++++++----- .../client/adapter/es/support/ESTemplate.java | 4 +- .../client/adapter/es/test/SqlParseTest.java | 7 +- .../adapter/es/test/sync/UserSyncTest.java | 17 +++++ .../src/test/resources/log4j2-test.xml | 13 ++++ .../src/main/resources/application.yml | 34 +++++----- 7 files changed, 110 insertions(+), 71 deletions(-) create mode 100644 client-adapter/elasticsearch/src/test/resources/log4j2-test.xml diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java index ccd58773..f0505aaa 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -23,7 +23,7 @@ public class SchemaItem { this.isAllFieldsSimple(); aliasTableItems.values().forEach(tableItem -> { tableItem.getRelationTableFields(); - tableItem.getRelationKeyFieldItems(); + // tableItem.getRelationKeyFieldItems(); tableItem.getRelationSelectFieldItems(); }); } @@ -145,8 +145,9 @@ public class SchemaItem { private boolean main; private boolean subQuery; - private volatile Map> relationTableFields; // 当前表关联条件字段 - private volatile List relationKeyFieldItems; // 关联条件字段在select中的对应字段 + private volatile Map> relationTableFields; // 当前表关联条件字段对应主表查询字段 + // private volatile List relationKeyFieldItems; // + // 关联条件字段在select中的对应字段 private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 public TableItem(SchemaItem schemaItem){ @@ -267,34 +268,36 @@ public class SchemaItem { return relationTableFields; } - public List getRelationKeyFieldItems() { - if (relationKeyFieldItems == null) { - synchronized (SchemaItem.class) { - if (relationKeyFieldItems == null) { - relationKeyFieldItems = new ArrayList<>(); - getRelationFields().forEach(relationFieldsPair -> { - FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); - List selectFieldItem = getSchemaItem().getColumnFields() - .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); - if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - relationKeyFieldItems.addAll(selectFieldItem); - } else { - FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); - selectFieldItem = getSchemaItem().getColumnFields() - .get(rightFieldItem.getOwner() + "." + rightFieldItem.getColumn().getColumnName()); - if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - relationKeyFieldItems.addAll(selectFieldItem); - } else { - throw new UnsupportedOperationException( - "Relation condition column must in select columns."); - } - } - }); - } - } - } - return relationKeyFieldItems; - } + // public List getRelationKeyFieldItems() { + // if (relationKeyFieldItems == null) { + // synchronized (SchemaItem.class) { + // if (relationKeyFieldItems == null) { + // relationKeyFieldItems = new ArrayList<>(); + // getRelationFields().forEach(relationFieldsPair -> { + // FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); + // List selectFieldItem = getSchemaItem().getColumnFields() + // .get(leftFieldItem.getOwner() + "." + + // leftFieldItem.getColumn().getColumnName()); + // if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + // relationKeyFieldItems.addAll(selectFieldItem); + // } else { + // FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + // selectFieldItem = getSchemaItem().getColumnFields() + // .get(rightFieldItem.getOwner() + "." + + // rightFieldItem.getColumn().getColumnName()); + // if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + // relationKeyFieldItems.addAll(selectFieldItem); + // } else { + // throw new UnsupportedOperationException( + // "Relation condition column must in select columns."); + // } + // } + // }); + // } + // } + // } + // return relationKeyFieldItems; + // } public List getRelationSelectFieldItems() { if (relationSelectFieldItems == null) { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 9eb69c12..4e86383e 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -190,41 +190,46 @@ public class ESSyncService { if (allFieldsSimple) { // 不是子查询 if (!tableItem.isSubQuery()) { - Map esFieldData = new LinkedHashMap<>(); for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { Object value = esTemplate .getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); esFieldData.put(fieldItem.getFieldName(), value); } + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Map.Entry> entry : tableItem.getRelationTableFields() .entrySet()) { Object value = esTemplate .getValFromData(mapping, data, entry.getKey().getColumn().getColumnName()); for (FieldItem fieldItem : entry.getValue()) { - queryBuilder.must(QueryBuilders.termsQuery(fieldItem.getFieldName(), value)); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); } } - // - // for (FieldItem fieldItem : tableItem.getRelationTableFields()) { - // Object value = esTemplate - // .getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); - // - // } - - // if (logger.isDebugEnabled()) { - // logger.debug("从表insert, 且均为简单字段，对es进行update_by_query, queryBuilder: {}, - // esFieldData:{}", queryBuilder.toString(), esFieldData); - // } + if (logger.isDebugEnabled()) { + logger.trace( + "Join table update es index by foreign key, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); - // if (!result) { - // logger.error("从表insert，均为简单字段，直接对es进行update_by_query, es更新存在错误, table: {}, - // index: {}, dml: {}", dml.getTable(), config.getEsSyn().getIndex(), dml); - // } + if (!result) { + logger.error( + "Join table update es index by foreign key error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } else { + // TODO } - // TODO } else { // TODO 查询总sql } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 9908c0d8..fec06dd1 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -264,7 +264,7 @@ public class ESTemplate { if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(itemResponse.getFailureMessage()); } else { - logger.error("ES 同步数据错误 {}", itemResponse.getFailureMessage()); + logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); } } } @@ -314,7 +314,7 @@ public class ESTemplate { value = resultSet.getByte(fieldName); } } - return value; + return ESSyncUtil.typeConvert(value, esType); } public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java index f31e6a89..d07defe9 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java @@ -39,8 +39,9 @@ public class SqlParseTest { .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); // 获取关联字段在select中的对应字段 - List relationSelectFieldItem = tableItem.getRelationKeyFieldItems(); - relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", - fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); + // List relationSelectFieldItem = + // tableItem.getRelationKeyFieldItems(); + // relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", + // fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java index 9fc9c211..c69fa995 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java @@ -3,6 +3,9 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync; import java.util.*; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -53,6 +56,7 @@ public class UserSyncTest { dataList.add(data); data.put("id", 1L); data.put("name", "Eric"); + data.put("role_id", 1L); data.put("c_time", new Date()); dml.setData(dataList); @@ -79,6 +83,7 @@ public class UserSyncTest { dataList.add(data); data.put("id", 1L); data.put("name", "Eric"); + data.put("role_id", 1L); data.put("c_time", new Date()); dml.setData(dataList); @@ -108,6 +113,18 @@ public class UserSyncTest { esAdapter.getEsSyncService().sync(dml); } + @Test + public void ttt(){ + SearchResponse response = esAdapter.getTransportClient().prepareSearch("mytest_user") + .setTypes("_doc") + .setQuery(QueryBuilders.termQuery("_id", 1L)) + .setSize(100) + .get(); + for (SearchHit hit : response.getHits()) { + System.out.println(hit.getSourceAsMap().get("name")); + } + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/resources/log4j2-test.xml b/client-adapter/elasticsearch/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..9c3d2ee6 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index a7d512c4..3e0e0dc4 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -16,34 +16,34 @@ hbase.zookeeper.property.clientPort: 2181 hbase.zookeeper.znode.parent: /hbase canal.conf: -# canalServerHost: 127.0.0.1:11111 + canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 - bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 +# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 flatMessage: true retry: 3 timeout: 20000 batchSize: 50 -# canalInstances: -# - instance: example -# groups: -# - outAdapters: -# - name: es -# hosts: 127.0.0.1:9300 -# properties: -# cluster.name: elasticsearch + canalInstances: + - instance: example + groups: + - outAdapters: + - name: es + hosts: 127.0.0.1:9300 + properties: + cluster.name: elasticsearch # - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} # hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} # zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} - mqTopics: - - mqMode: kafka - topic: example - groups: - - groupId: g2 - outAdapters: - - name: logger +# mqTopics: +# - mqMode: kafka +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger # mqTopics: # - mqMode: rocketmq # topic: example From b2b44555cafde06811bcbde4b2ff555767d6a4e2 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 11:07:35 +0800 Subject: [PATCH 18/42] =?UTF-8?q?=E4=BB=8E=E8=A1=A8=E6=8F=92=E5=85=A5,=20?= =?UTF-8?q?=E4=B8=BB=E8=A1=A8=E4=B8=BA=E7=AE=80=E5=8D=95=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/adapter/es/config/SchemaItem.java | 62 ++------ .../adapter/es/service/ESSyncService.java | 125 ++++++++++++++-- .../client/adapter/es/support/ESSyncUtil.java | 4 +- .../client/adapter/es/support/ESTemplate.java | 84 ++--------- .../client/adapter/es/test/sync/Common.java | 27 ++++ .../es/test/sync/RoleSyncJoinOneTest.java | 57 ++++++++ .../es/test/sync/RoleSyncJoinSubTest.java | 57 ++++++++ .../es/test/sync/UserSyncJoinOneTest.java | 59 ++++++++ .../es/test/sync/UserSyncSingleTest.java | 75 ++++++++++ .../adapter/es/test/sync/UserSyncTest.java | 133 ------------------ .../resources/es/mytest_user_join_one.yml | 10 ++ .../resources/es/mytest_user_join_sub.yml | 10 ++ .../test/resources/es/mytest_user_single.yml | 8 ++ 13 files changed, 447 insertions(+), 264 deletions(-) create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java delete mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java create mode 100644 client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml create mode 100644 client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml create mode 100644 client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java index f0505aaa..f092bf78 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -23,7 +23,6 @@ public class SchemaItem { this.isAllFieldsSimple(); aliasTableItems.values().forEach(tableItem -> { tableItem.getRelationTableFields(); - // tableItem.getRelationKeyFieldItems(); tableItem.getRelationSelectFieldItems(); }); } @@ -76,19 +75,19 @@ public class SchemaItem { getSelectFields() .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); - if (!tableItem.isSubQuery()) { - List fieldItems = columnFields.computeIfAbsent( - columnItem.getOwner() + "." + columnItem.getColumnName(), - k -> new ArrayList<>()); - fieldItems.add(fieldItem); - } else { - tableItem.getSubQueryFields().forEach(subQueryField -> { - List fieldItems = columnFields.computeIfAbsent( - columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), - k -> new ArrayList<>()); - fieldItems.add(fieldItem); - }); - } + // if (!tableItem.isSubQuery()) { + List fieldItems = columnFields.computeIfAbsent( + columnItem.getOwner() + "." + columnItem.getColumnName(), + k -> new ArrayList<>()); + fieldItems.add(fieldItem); + // } else { + // tableItem.getSubQueryFields().forEach(subQueryField -> { + // List fieldItems = columnFields.computeIfAbsent( + // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), + // k -> new ArrayList<>()); + // fieldItems.add(fieldItem); + // }); + // } })); } } @@ -146,8 +145,6 @@ public class SchemaItem { private boolean subQuery; private volatile Map> relationTableFields; // 当前表关联条件字段对应主表查询字段 - // private volatile List relationKeyFieldItems; // - // 关联条件字段在select中的对应字段 private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 public TableItem(SchemaItem schemaItem){ @@ -237,10 +234,8 @@ public class SchemaItem { FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); FieldItem currentTableRelField = null; if (getAlias().equals(leftFieldItem.getOwner())) { - // relationTableFields.add(leftFieldItem); currentTableRelField = leftFieldItem; } else if (getAlias().equals(rightFieldItem.getOwner())) { - // relationTableFields.add(rightFieldItem); currentTableRelField = rightFieldItem; } @@ -268,37 +263,6 @@ public class SchemaItem { return relationTableFields; } - // public List getRelationKeyFieldItems() { - // if (relationKeyFieldItems == null) { - // synchronized (SchemaItem.class) { - // if (relationKeyFieldItems == null) { - // relationKeyFieldItems = new ArrayList<>(); - // getRelationFields().forEach(relationFieldsPair -> { - // FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); - // List selectFieldItem = getSchemaItem().getColumnFields() - // .get(leftFieldItem.getOwner() + "." + - // leftFieldItem.getColumn().getColumnName()); - // if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - // relationKeyFieldItems.addAll(selectFieldItem); - // } else { - // FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); - // selectFieldItem = getSchemaItem().getColumnFields() - // .get(rightFieldItem.getOwner() + "." + - // rightFieldItem.getColumn().getColumnName()); - // if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - // relationKeyFieldItems.addAll(selectFieldItem); - // } else { - // throw new UnsupportedOperationException( - // "Relation condition column must in select columns."); - // } - // } - // }); - // } - // } - // } - // return relationKeyFieldItems; - // } - public List getRelationSelectFieldItems() { if (relationSelectFieldItems == null) { synchronized (SchemaItem.class) { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 4e86383e..9cdff29a 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -133,12 +133,20 @@ public class ESSyncService { continue; // 单表插入完成 } - // ------是否主表 查询sql来插入------ + // ------是主表 查询sql来插入------ if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { String sql = mapping.getSql(); String condition = ESSyncUtil.pkConditionSql(mapping, data); sql = ESSyncUtil.appendCondition(sql, condition); DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace( + "Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } ESSyncUtil.sqlRS(ds, sql, rs -> { try { while (rs.next()) { @@ -192,23 +200,30 @@ public class ESSyncService { if (!tableItem.isSubQuery()) { Map esFieldData = new LinkedHashMap<>(); for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object value = esTemplate - .getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); + Object value = esTemplate.getValFromData(mapping, + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); esFieldData.put(fieldItem.getFieldName(), value); } BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Map.Entry> entry : tableItem.getRelationTableFields() .entrySet()) { - Object value = esTemplate - .getValFromData(mapping, data, entry.getKey().getColumn().getColumnName()); for (FieldItem fieldItem : entry.getValue()) { - String fieldName = fieldItem.getFieldName(); - // 判断是否是主键 - if (fieldName.equals(mapping.get_id())) { - fieldName = "_id"; + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromData(mapping, + data, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); } } @@ -228,7 +243,95 @@ public class ESSyncService { mapping.get_index()); } } else { - // TODO + StringBuilder sql = new StringBuilder( + "SELECT * FROM (" + tableItem.getSubQuerySql() + ") " + tableItem.getAlias() + " WHERE "); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate + .getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + if (value instanceof String) { + sql.append(tableItem.getAlias()) + .append(".") + .append(columnName) + .append("='") + .append(value) + .append("' ") + .append(" AND "); + } else { + sql.append(tableItem.getAlias()) + .append(".") + .append(columnName) + .append("=") + .append(value) + .append(" ") + .append(" AND "); + } + } + int len = sql.length(); + sql.delete(len - 5, len); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace( + "Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), val); + } + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Map.Entry> entry : tableItem + .getRelationTableFields() + .entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace( + "Join table update es index by foreign key, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + if (!result) { + logger.error( + "Join table update es index by foreign key error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); } } else { // TODO 查询总sql diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index f09641d7..88f7b496 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -2,7 +2,6 @@ package com.alibaba.otter.canal.client.adapter.es.support; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.*; @@ -20,6 +19,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; public class ESSyncUtil { @@ -249,7 +249,7 @@ public class ESSyncUtil { /** * 执行查询sql */ - public static Object sqlRS(DataSource ds, String sql, Function fun) { + public static Object sqlRS(DataSource ds, String sql, Function fun) { Connection conn = null; Statement smt = null; ResultSet rs = null; diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index fec06dd1..7df0c77e 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -274,44 +274,14 @@ public class ESTemplate { return true; } - // public Object getRSDataValue(ESMapping mapping, ResultSet rs, String - // fieldName) throws SQLException { - // String esType = getEsType(mapping, fieldName); - // Object value = rs.getObject(fieldName); - // if (value instanceof Boolean) { - // // 判断es类型 - // if (!"boolean".equals(esType)) { - // value = rs.getByte(fieldName); - // } - // } - // return value; - // } - // - // public Object getDataValue(ESMapping mapping, Map data, - // String fieldName) { - // String esType = getEsType(mapping, fieldName); - // Object value = data.get(fieldName); - // if (value instanceof Byte) { - // if ("boolean".equals(esType)) { - // value = ((Byte) value).intValue() != 0; - // } - // } - // return value; - // } - - // public Object convertType(ESMapping mapping, String fieldName, Object val) { - // // 从mapping中获取类型来转换 - // String esType = getEsType(mapping, fieldName); - // return ESSyncUtil.typeConvert(val, esType); - // } - - public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName) throws SQLException { + public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException { String esType = getEsType(mapping, fieldName); - Object value = resultSet.getObject(fieldName); + Object value = resultSet.getObject(columnName); if (value instanceof Boolean) { if (!"boolean".equals(esType)) { - value = resultSet.getByte(fieldName); + value = resultSet.getByte(columnName); } } return ESSyncUtil.typeConvert(value, esType); @@ -323,48 +293,23 @@ public class ESTemplate { String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); Object resultIdVal = null; for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - String fieldName = Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); - Object value = getValFromRS(mapping, resultSet, fieldName); + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); if (fieldItem.getFieldName().equals(idFieldName)) { resultIdVal = value; } - if (!fieldItem.getFieldName().equals(mapping.get_id()) && !mapping.getSkips().contains(fieldName)) { - esFieldData.put(fieldName, value); + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); } } return resultIdVal; } - // public Object getIdValFromDmlData(ESMapping mapping, Map - // dmlData) { - // SchemaItem schemaItem = mapping.getSchemaItem(); - // String idFieldName = mapping.get_id() == null ? mapping.getPk() : - // mapping.get_id(); - // - // for (SchemaItem.FieldItem fieldItem : schemaItem.getSelectFields().values()) - // { - // if (fieldItem.getFieldName().equals(idFieldName)) { - // String fieldName = - // Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); - // String esType = getEsType(mapping, fieldName); - // Object value = dmlData.get(fieldName); - // if (value instanceof Byte) { - // if ("boolean".equals(esType)) { - // value = ((Byte) value).intValue() != 0; - // } - // } - // - // return ESSyncUtil.typeConvert(value, esType); - // } - // } - // return null; - // } - - public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName) { + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { String esType = getEsType(mapping, fieldName); - Object value = dmlData.get(fieldName); + Object value = dmlData.get(columnName); if (value instanceof Byte) { if ("boolean".equals(esType)) { value = ((Byte) value).intValue() != 0; @@ -387,15 +332,16 @@ public class ESTemplate { String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); Object resultIdVal = null; for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - String fieldName = Util.cleanColumn(fieldItem.getColumnItems().iterator().next().getColumnName()); - Object value = getValFromData(mapping, dmlData, fieldName); + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); if (fieldItem.getFieldName().equals(idFieldName)) { resultIdVal = value; } - if (!fieldItem.getFieldName().equals(mapping.get_id()) && !mapping.getSkips().contains(fieldName)) { - esFieldData.put(fieldName, value); + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); } } return resultIdVal; diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java new file mode 100644 index 00000000..e5b31cbd --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.es.test.TestConstant; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +import java.util.HashMap; +import java.util.Map; + +public class Common { + + public static ESAdapter init() { + DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); + + OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); + outerAdapterConfig.setName("es"); + outerAdapterConfig.setHosts(TestConstant.esHosts); + Map properties = new HashMap<>(); + properties.put("cluster.name", TestConstant.clusterNmae); + outerAdapterConfig.setProperties(properties); + + ESAdapter esAdapter = new ESAdapter(); + esAdapter.init(outerAdapterConfig); + return esAdapter; + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java new file mode 100644 index 00000000..dde3351d --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class RoleSyncJoinOneTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user_join_one.yml"); + esAdapter = Common.init(); + } + + /** + * 非子查询从表插入 (确保主表记录必须有数据) + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin", response.getSource().get("_role_name")); + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java new file mode 100644 index 00000000..31387898 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class RoleSyncJoinSubTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user_join_sub.yml"); + esAdapter = Common.init(); + } + + /** + * 非子查询从表插入 (确保主表记录必须有数据) + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin2"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin2", response.getSource().get("_role_name")); + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java new file mode 100644 index 00000000..f2f9f4e3 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java @@ -0,0 +1,59 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class UserSyncJoinOneTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user_join_one.yml"); + esAdapter = Common.init(); + } + + /** + * 主表带函数插入, 数据库里内容必须和单测一致 + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric_", response.getSource().get("_name")); + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java new file mode 100644 index 00000000..16a25bea --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java @@ -0,0 +1,75 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class UserSyncSingleTest { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user_single.yml"); + esAdapter = Common.init(); + } + + /** + * 单表插入 + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric", response.getSource().get("_name")); + } + + @Test + public void ttt() { + SearchResponse response = esAdapter.getTransportClient() + .prepareSearch("mytest_user") + .setTypes("_doc") + .setQuery(QueryBuilders.termQuery("_id", 1L)) + .setSize(100) + .get(); + for (SearchHit hit : response.getHits()) { + System.out.println(hit.getSourceAsMap().get("name")); + } + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java deleted file mode 100644 index c69fa995..00000000 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.es.test.sync; - -import java.util.*; - -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.otter.canal.client.adapter.es.ESAdapter; -import com.alibaba.otter.canal.client.adapter.es.test.TestConstant; -import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; - -public class UserSyncTest { - - private ESAdapter esAdapter; - - @Before - public void init() { - AdapterConfigs.put("es", "mytest_user.yml"); - DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); - - OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); - outerAdapterConfig.setName("es"); - outerAdapterConfig.setHosts(TestConstant.esHosts); - Map properties = new HashMap<>(); - properties.put("cluster.name", TestConstant.clusterNmae); - outerAdapterConfig.setProperties(properties); - - esAdapter = new ESAdapter(); - esAdapter.init(outerAdapterConfig); - - } - - /** - * 单表插入 - */ - @Test - public void insertTest01() { - Dml dml = new Dml(); - dml.setDestination("example"); - dml.setTs(new Date().getTime()); - dml.setType("INSERT"); - dml.setDatabase("mytest"); - dml.setTable("user"); - List> dataList = new ArrayList<>(); - Map data = new LinkedHashMap<>(); - dataList.add(data); - data.put("id", 1L); - data.put("name", "Eric"); - data.put("role_id", 1L); - data.put("c_time", new Date()); - - dml.setData(dataList); - - esAdapter.getEsSyncService().sync(dml); - - GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("Eric", response.getSource().get("name")); - } - - /** - * 主表带函数插入, 数据库里内容必须和单测一致 - */ - @Test - public void insertTest02() { - Dml dml = new Dml(); - dml.setDestination("example"); - dml.setTs(new Date().getTime()); - dml.setType("INSERT"); - dml.setDatabase("mytest"); - dml.setTable("user"); - List> dataList = new ArrayList<>(); - Map data = new LinkedHashMap<>(); - dataList.add(data); - data.put("id", 1L); - data.put("name", "Eric"); - data.put("role_id", 1L); - data.put("c_time", new Date()); - - dml.setData(dataList); - - esAdapter.getEsSyncService().sync(dml); - - GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("Eric_", response.getSource().get("name")); - } - - @Test - public void insertTest03() { - Dml dml = new Dml(); - dml.setDestination("example"); - dml.setTs(new Date().getTime()); - dml.setType("INSERT"); - dml.setDatabase("mytest"); - dml.setTable("role"); - List> dataList = new ArrayList<>(); - Map data = new LinkedHashMap<>(); - dataList.add(data); - data.put("id", 1L); - data.put("role_name", "admin"); - - dml.setData(dataList); - - esAdapter.getEsSyncService().sync(dml); - } - - @Test - public void ttt(){ - SearchResponse response = esAdapter.getTransportClient().prepareSearch("mytest_user") - .setTypes("_doc") - .setQuery(QueryBuilders.termQuery("_id", 1L)) - .setSize(100) - .get(); - for (SearchHit hit : response.getHits()) { - System.out.println(hit.getSourceAsMap().get("name")); - } - } - - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); - } -} diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml new file mode 100644 index 00000000..8e2f4efa --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml @@ -0,0 +1,10 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, + b.role_name as _role_name, a.c_time as _c_time from user a + left join role b on b.id=a.role_id" + commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml new file mode 100644 index 00000000..bde1139f --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml @@ -0,0 +1,10 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, + b.role_name as _role_name, a.c_time as _c_time from user a + left join (select id, role_name from role ) b on b.id=a.role_id" + commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml new file mode 100644 index 00000000..98fad232 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml @@ -0,0 +1,8 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, a.names as _name, a.role_id as _role_id, a.c_time as _c_time from user a" + commitBatch: 3000 \ No newline at end of file From c9ad18894dc8024ebeb09af571f20d0c324581b9 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 11:33:12 +0800 Subject: [PATCH 19/42] =?UTF-8?q?=E4=BB=8E=E8=A1=A8=E6=8F=92=E5=85=A5,=20?= =?UTF-8?q?=E4=B8=BB=E8=A1=A8=E4=B8=BA=E7=AE=80=E5=8D=95=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/adapter/es/test/sync/RoleSyncJoinSubTest.java | 9 +++++---- .../src/test/resources/es/mytest_user_join_sub.yml | 5 +++-- .../src/test/resources/es/mytest_user_single.yml | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java index 31387898..c56d1d3c 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java @@ -25,7 +25,7 @@ public class RoleSyncJoinSubTest { } /** - * 非子查询从表插入 (确保主表记录必须有数据) + * 子查询从表插入 (确保主表记录必须有数据) */ @Test public void insertTest01() { @@ -34,19 +34,20 @@ public class RoleSyncJoinSubTest { dml.setTs(new Date().getTime()); dml.setType("INSERT"); dml.setDatabase("mytest"); - dml.setTable("role"); + dml.setTable("label"); List> dataList = new ArrayList<>(); Map data = new LinkedHashMap<>(); dataList.add(data); data.put("id", 1L); - data.put("role_name", "admin2"); + data.put("user_id",1L); + data.put("label", "a"); dml.setData(dataList); esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("admin2", response.getSource().get("_role_name")); + Assert.assertEquals("a;b", response.getSource().get("_labels")); } @After diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml index bde1139f..2583d40d 100644 --- a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml @@ -5,6 +5,7 @@ esMapping: _type: _doc _id: _id sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, - b.role_name as _role_name, a.c_time as _c_time from user a - left join (select id, role_name from role ) b on b.id=a.role_id" + b.labels _labels, a.c_time as _c_time from user a + left join (select user_id, group_concat(label separator ';') as labels from label + group by user_id) b on b.user_id=a.id" commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml index 98fad232..23e1b491 100644 --- a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml @@ -4,5 +4,5 @@ esMapping: _index: mytest_user _type: _doc _id: _id - sql: "select a.id as _id, a.names as _name, a.role_id as _role_id, a.c_time as _c_time from user a" + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a" commitBatch: 3000 \ No newline at end of file From 9d04a82c4284aa97d1bbfdbe0a68f6165393596b Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 14:32:11 +0800 Subject: [PATCH 20/42] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/es/service/ESSyncService.java | 571 +++++++++++------- .../client/adapter/es/support/ESTemplate.java | 6 +- .../es/test/sync/RoleSyncJoinOne2Test.java | 57 ++ .../es/test/sync/RoleSyncJoinSub2Test.java | 58 ++ .../resources/es/mytest_user_join_one2.yml | 10 + .../resources/es/mytest_user_join_sub2.yml | 11 + 6 files changed, 494 insertions(+), 219 deletions(-) create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java create mode 100644 client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one2.yml create mode 100644 client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 9cdff29a..ee531e91 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -98,12 +98,17 @@ public class ESSyncService { } } + /** + * 插入dml操作 + * + * @param config es配置 + * @param dml dml数据 + */ private void insert(ESSyncConfig config, Dml dml) { List> dataList = dml.getData(); if (dataList == null || dataList.isEmpty()) { return; } - ESMapping mapping = config.getEsMapping(); SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); for (Map data : dataList) { if (data == null || data.isEmpty()) { @@ -112,231 +117,365 @@ public class ESSyncService { // ------是否单表 & 所有字段都为简单字段------ if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { - Map esFieldData = new LinkedHashMap<>(); - Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData); - - if (logger.isTraceEnabled()) { - logger.trace("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index(), - idVal); + singleTableSimpleFiledInsert(config, dml, data); + } else { + // ------是主表 查询sql来插入------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + mainTableInsert(config, dml, data); } - boolean result = esTemplate.insert(config, idVal, esFieldData); - if (!result) { - logger.error("Single table insert to es index error, destination:{}, table: {}, index: {}, id: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index(), - idVal); - } - continue; // 单表插入完成 - } - // ------是主表 查询sql来插入------ - if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { - String sql = mapping.getSql(); - String condition = ESSyncUtil.pkConditionSql(mapping, data); - sql = ESSyncUtil.appendCondition(sql, condition); - DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); - if (logger.isTraceEnabled()) { - logger.trace( - "Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index(), - sql.replace("\n", " ")); - } - ESSyncUtil.sqlRS(ds, sql, rs -> { - try { - while (rs.next()) { - Map esFieldData = new LinkedHashMap<>(); - Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData); - - if (logger.isTraceEnabled()) { - logger.trace( - "Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index(), - idVal); - } - boolean result = esTemplate.insert(config, idVal, esFieldData); - if (!result) { - logger.error( - "Single table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index(), - idVal); - } - } - } catch (Exception e) { - throw new RuntimeException(e); + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; } - return 0; - }); - } - - // 从表的操作 - for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { - if (tableItem.isMain()) { - continue; - } - if (!tableItem.getTableName().equals(dml.getTable())) { - continue; - } - // ------关联条件出现在主表查询条件------ - boolean allFieldsSimple = true; - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { - allFieldsSimple = false; - break; + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; } - } - // 所有查询字段均为简单字段 - if (allFieldsSimple) { - // 不是子查询 - if (!tableItem.isSubQuery()) { - Map esFieldData = new LinkedHashMap<>(); - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object value = esTemplate.getValFromData(mapping, - data, - fieldItem.getFieldName(), - fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), value); + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; } - - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Map.Entry> entry : tableItem.getRelationTableFields() - .entrySet()) { - for (FieldItem fieldItem : entry.getValue()) { - if (fieldItem.getColumnItems().size() == 1) { - Object value = esTemplate.getValFromData(mapping, - data, - fieldItem.getFieldName(), - entry.getKey().getColumn().getColumnName()); - - String fieldName = fieldItem.getFieldName(); - // 判断是否是主键 - if (fieldName.equals(mapping.get_id())) { - fieldName = "_id"; - } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); - } - } - } - - if (logger.isDebugEnabled()) { - logger.trace( - "Join table update es index by foreign key, destination:{}, table: {}, index: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index()); - } - boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); - if (!result) { - logger.error( - "Join table update es index by foreign key error, destination:{}, table: {}, index: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index()); + } + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段插入------ + joinTableSimpleFieldOperation(config, dml, data, tableItem); + } else { + // ------关联子表简单字段插入------ + subTableSimpleFieldOperation(config, dml, data, tableItem); } } else { - StringBuilder sql = new StringBuilder( - "SELECT * FROM (" + tableItem.getSubQuerySql() + ") " + tableItem.getAlias() + " WHERE "); - - for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { - String columnName = fkFieldItem.getColumn().getColumnName(); - Object value = esTemplate - .getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); - if (value instanceof String) { - sql.append(tableItem.getAlias()) - .append(".") - .append(columnName) - .append("='") - .append(value) - .append("' ") - .append(" AND "); - } else { - sql.append(tableItem.getAlias()) - .append(".") - .append(columnName) - .append("=") - .append(value) - .append(" ") - .append(" AND "); - } - } - int len = sql.length(); - sql.delete(len - 5, len); - DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); - if (logger.isTraceEnabled()) { - logger.trace( - "Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index(), - sql.toString().replace("\n", " ")); - } - ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { - try { - while (rs.next()) { - Map esFieldData = new LinkedHashMap<>(); - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object val = esTemplate.getValFromRS(mapping, - rs, - fieldItem.getFieldName(), - fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), val); - } - - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - for (Map.Entry> entry : tableItem - .getRelationTableFields() - .entrySet()) { - for (FieldItem fieldItem : entry.getValue()) { - if (fieldItem.getColumnItems().size() == 1) { - Object value = esTemplate.getValFromRS(mapping, - rs, - fieldItem.getFieldName(), - entry.getKey().getColumn().getColumnName()); - String fieldName = fieldItem.getFieldName(); - // 判断是否是主键 - if (fieldName.equals(mapping.get_id())) { - fieldName = "_id"; - } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); - } - } - } - - if (logger.isDebugEnabled()) { - logger.trace( - "Join table update es index by foreign key, destination:{}, table: {}, index: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index()); - } - boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); - if (!result) { - logger.error( - "Join table update es index by foreign key error, destination:{}, table: {}, index: {}", - config.getDestination(), - dml.getTable(), - mapping.get_index()); - } - } - } catch (Exception e) { - throw new RuntimeException(e); - } - return 0; - }); + // ------关联子表复杂字段插入 执行全sql更新es------ + jonTableWholeSqlOperation(config, dml, data, tableItem); } - } else { - // TODO 查询总sql } } } } + + /** + * 单表简单字段insert + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.insert(config, idVal, esFieldData); + if (!result) { + logger.error("Single table insert to es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + + /** + * 主表(单表)复杂字段insert + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void mainTableInsert(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.insert(config, idVal, esFieldData); + if (!result) { + logger.error( + "Single table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 关联表主表简单字段operation + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, + TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate + .getValFromData(mapping, data, fieldItem.getFieldName(), fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromData(mapping, + data, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + if (!result) { + logger.error("Join table update es index by foreign key error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + + /** + * 关联子查询, 主表简单字段operation + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, + TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + StringBuilder sql = new StringBuilder( + "SELECT * FROM (" + tableItem.getSubQuerySql() + ") " + tableItem.getAlias() + " WHERE "); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + if (value instanceof String) { + sql.append(tableItem.getAlias()) + .append(".") + .append(columnName) + .append("='") + .append(value) + .append("' ") + .append(" AND "); + } else { + sql.append(tableItem.getAlias()) + .append(".") + .append(columnName) + .append("=") + .append(value) + .append(" ") + .append(" AND "); + } + } + int len = sql.length(); + sql.delete(len - 5, len); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object val = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), val); + } + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + entry.getKey().getColumn().getColumnName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + if (!result) { + logger.error( + "Join table update es index by query sql error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + + /** + * 关联(子查询), 主表复杂字段operation, 全sql执行 + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + * @param tableItem 当前表配置 + */ + private void jonTableWholeSqlOperation(ESSyncConfig config, Dml dml, Map data, + TableItem tableItem) { + ESMapping mapping = config.getEsMapping(); + StringBuilder sql = new StringBuilder(mapping.getSql() + " WHERE "); + + for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { + String columnName = fkFieldItem.getColumn().getColumnName(); + Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); + if (value instanceof String) { + sql.append(tableItem.getAlias()) + .append(".") + .append(columnName) + .append("='") + .append(value) + .append("' ") + .append(" AND "); + } else { + sql.append(tableItem.getAlias()) + .append(".") + .append(columnName) + .append("=") + .append(value) + .append(" ") + .append(" AND "); + } + } + int len = sql.length(); + sql.delete(len - 5, len); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.toString().replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object val = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + } + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { + for (FieldItem fieldItem : entry.getValue()) { + if (fieldItem.getColumnItems().size() == 1) { + Object value = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; + } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + } + } + } + + if (logger.isDebugEnabled()) { + logger.trace( + "Join table update es index by query whole sql, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + if (!result) { + logger.error( + "Join table update es index by query whole sql error, destination:{}, table: {}, index: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 7df0c77e..e81935ab 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -142,7 +142,7 @@ public class ESTemplate { return updateByQuery(mapping, queryBuilder, esFieldData, 1); } - public boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, + private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, int counter) { if (CollectionUtils.isEmpty(esFieldData)) { return true; @@ -204,10 +204,10 @@ public class ESTemplate { if (response.getStatus().getVersionConflicts() > 0) { if (counter >= 3) { - logger.error("第 {} 次执行updateByQuery, 依旧存在版本冲突，不再继续重试。", counter); + logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter); return false; } - logger.warn("本次updateByQuery存在版本冲突，准备重新执行..."); + logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行..."); try { TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java new file mode 100644 index 00000000..577ec758 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class RoleSyncJoinOne2Test { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user_join_one2.yml"); + esAdapter = Common.init(); + } + + /** + * 非子查询从表插入 (确保主表记录必须有数据) + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin2"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin2_", response.getSource().get("_role_name")); + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java new file mode 100644 index 00000000..4d67139b --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.client.adapter.es.test.sync; + +import java.util.*; + +import org.elasticsearch.action.get.GetResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.ESAdapter; +import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.Dml; + +public class RoleSyncJoinSub2Test { + + private ESAdapter esAdapter; + + @Before + public void init() { + AdapterConfigs.put("es", "mytest_user_join_sub2.yml"); + esAdapter = Common.init(); + } + + /** + * 子查询从表插入 (确保主表记录必须有数据) + */ + @Test + public void insertTest01() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("INSERT"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id",1L); + data.put("label", "a"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("a;b_", response.getSource().get("_labels")); + } + + @After + public void after() { + esAdapter.destroy(); + DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + } +} diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one2.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one2.yml new file mode 100644 index 00000000..4a44c5e0 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one2.yml @@ -0,0 +1,10 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, + concat(b.role_name,'_') as _role_name, a.c_time as _c_time from user a + left join role b on b.id=a.role_id" + commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml new file mode 100644 index 00000000..8ae06ab4 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml @@ -0,0 +1,11 @@ +dataSourceKey: defaultDS +destination: example +esMapping: + _index: mytest_user + _type: _doc + _id: _id + sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, + concat(b.labels, '_') as _labels, a.c_time as _c_time from user a + left join (select user_id, group_concat(label separator ';') as labels from label + group by user_id) b on b.user_id=a.id" + commitBatch: 3000 \ No newline at end of file From a6b16eedfaa156be4ac88c9c61bf10357f3a11b4 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 14:43:59 +0800 Subject: [PATCH 21/42] modify --- .../client/adapter/es/service/ESSyncService.java | 16 ++++++++++++---- .../adapter/hbase/service/HbaseSyncService.java | 6 +++--- .../adapter/logger/LoggerAdapterExample.java | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index ee531e91..f34e38e1 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -65,7 +65,7 @@ public class ESSyncService { } } if (logger.isTraceEnabled()) { - logger.trace("Sync elapsed time: {}, effect index count：{}, destination: {}", + logger.trace("Sync elapsed time: {} ms, effect index count：{}, destination: {}", (System.currentTimeMillis() - begin), esSyncConfigs.size(), dml.getDestination()); @@ -81,13 +81,13 @@ public class ESSyncService { if (type != null && type.equalsIgnoreCase("INSERT")) { insert(config, dml); } else if (type != null && type.equalsIgnoreCase("UPDATE")) { - // update(config, dml); + update(config, dml); } else if (type != null && type.equalsIgnoreCase("DELETE")) { - // delete(config, dml); + delete(config, dml); } if (logger.isTraceEnabled()) { - logger.trace("Sync elapsed time: {},destination: {}, es index: {}", + logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}", (System.currentTimeMillis() - begin), dml.getDestination(), config.getEsMapping().get_index()); @@ -98,6 +98,14 @@ public class ESSyncService { } } + private void delete(ESSyncConfig config, Dml dml) { + + } + + private void update(ESSyncConfig config, Dml dml) { + + } + /** * 插入dml操作 * diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 3dd03b91..9606ee97 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -38,8 +38,7 @@ public class HbaseSyncService { delete(config, dml); } if (logger.isDebugEnabled()) { - String res = dml.toString(); - logger.debug(res); + logger.debug("DML: {}", dml.toString()); } } } catch (Exception e) { @@ -199,7 +198,8 @@ public class HbaseSyncService { Map columnItems = hbaseMapping.getColumnItems(); HRow hRow = new HRow(rowKeyBytes); for (String updateColumn : old.get(index).keySet()) { - if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(updateColumn)) { + if (hbaseMapping.getExcludeColumns() != null + && hbaseMapping.getExcludeColumns().contains(updateColumn)) { continue; } MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index 985c7804..c7b7978d 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -27,7 +27,7 @@ public class LoggerAdapterExample implements OuterAdapter { @Override public void sync(Dml dml) { - logger.info(dml.toString()); + logger.info("DML: ", dml.toString()); } @Override From b9c4884af137e92523b13c2799d1acca5f5d36a5 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 14:46:29 +0800 Subject: [PATCH 22/42] fix #1083 --- .../canal/rocketmq/CanalRocketMQProducer.java | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java index aaace132..1cc6b109 100644 --- a/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java +++ b/server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java @@ -77,8 +77,8 @@ public class CanalRocketMQProducer implements CanalMQProducer { JSON.toJSONString(flatMessage), destination.getTopic(), destination.getPartition()); - Message message = new Message(destination.getTopic(), JSON.toJSONString(flatMessage) - .getBytes()); + Message message = new Message(destination.getTopic(), + JSON.toJSONString(flatMessage).getBytes()); this.defaultMQProducer.send(message, new MessageQueueSelector() { @Override @@ -98,28 +98,32 @@ public class CanalRocketMQProducer implements CanalMQProducer { int length = partitionFlatMessage.length; for (int i = 0; i < length; i++) { FlatMessage flatMessagePart = partitionFlatMessage[i]; - logger.debug("flatMessagePart: {}, partition: {}", - JSON.toJSONString(flatMessagePart), - i); - final int index = i; - try { - Message message = new Message(destination.getTopic(), - JSON.toJSONString(flatMessagePart).getBytes()); - this.defaultMQProducer.send(message, new MessageQueueSelector() { + if (flatMessagePart != null) { + logger.debug("flatMessagePart: {}, partition: {}", + JSON.toJSONString(flatMessagePart), + i); + final int index = i; + try { + Message message = new Message(destination.getTopic(), + JSON.toJSONString(flatMessagePart).getBytes()); + this.defaultMQProducer.send(message, new MessageQueueSelector() { - @Override - public MessageQueue select(List mqs, Message msg, Object arg) { - if (index > mqs.size()) { - throw new CanalServerException("partition number is error,config num:" - + destination.getPartitionsNum() - + ", mq num: " + mqs.size()); + @Override + public MessageQueue select(List mqs, Message msg, + Object arg) { + if (index > mqs.size()) { + throw new CanalServerException( + "partition number is error,config num:" + + destination.getPartitionsNum() + + ", mq num: " + mqs.size()); + } + return mqs.get(index); } - return mqs.get(index); - } - }, null); - } catch (Exception e) { - logger.error("send flat message to hashed partition error", e); - callback.rollback(); + }, null); + } catch (Exception e) { + logger.error("send flat message to hashed partition error", e); + callback.rollback(); + } } } } From 865ba94605ccd029f6c81d2990808ec907172e3e Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 15:22:14 +0800 Subject: [PATCH 23/42] =?UTF-8?q?application.yml=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=89=93=E5=8C=85=E8=AE=BE=E7=BD=AE=E9=BB=98=E8=AE=A4=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/launcher/pom.xml | 8 --- .../src/main/resources/application-sample.yml | 49 +++++++++++++++++++ .../src/main/resources/application.yml | 46 +---------------- 3 files changed, 50 insertions(+), 53 deletions(-) create mode 100644 client-adapter/launcher/src/main/resources/application-sample.yml diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 708e1471..1a79ed21 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -166,14 +166,6 @@ - - - src/main/resources - - application.yml - - - diff --git a/client-adapter/launcher/src/main/resources/application-sample.yml b/client-adapter/launcher/src/main/resources/application-sample.yml new file mode 100644 index 00000000..706e4088 --- /dev/null +++ b/client-adapter/launcher/src/main/resources/application-sample.yml @@ -0,0 +1,49 @@ +server: + port: 8081 +logging: + level: + com.alibaba.otter.canal.client.adapter.hbase: DEBUG +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + time-zone: GMT+8 + default-property-inclusion: non_null + +canal.conf: + canalServerHost: 127.0.0.1:11111 +# zookeeperHosts: slave1:2181 +# bootstrapServers: slave1:6667 #or rocketmq +# flatMessage: true + canalInstances: + - instance: example + groups: + - outAdapters: + - name: logger +# - name: hbase +# properties: +# hbase.zookeeper.quorum: 127.0.0.1 +# hbase.zookeeper.property.clientPort: 2181 +# zookeeper.znode.parent: /hbase +# mqTopics: +# - mqMode: kafka +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger +# mqTopics: +# - mqMode: rocketmq +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger +# +#adapter.conf: +# datasourceConfigs: +# defaultDS: +# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true +# username: root +# password: 121212 +# adapterConfigs: +# - hbase/mytest_person2.yml diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 250d6a60..7fe7b191 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -7,48 +7,4 @@ spring: jackson: date-format: yyyy-MM-dd HH:mm:ss time-zone: GMT+8 - default-property-inclusion: non_null - - -hbasezookeeper.quorum: 127.0.0.1 -hbase.zookeeper.property.clientPort: 2181 -hbase.zookeeper.znode.parent: /hbase - -canal.conf: - canalServerHost: 127.0.0.1:11111 -# zookeeperHosts: slave1:2181 -# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 - flatMessage: true - canalInstances: - - instance: example - groups: - - outAdapters: - - name: logger -# - name: hbase -# properties: -# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} -# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} -# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} -# mqTopics: -# - mqMode: kafka -# topic: example -# groups: -# - groupId: g2 -# outAdapters: -# - name: logger -# mqTopics: -# - mqMode: rocketmq -# topic: example -# groups: -# - groupId: g2 -# outAdapters: -# - name: logger - -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: -# - hbase/mytest_person2.yml + default-property-inclusion: non_null \ No newline at end of file From 3bb408194ddcf39de74af2c94a43537469dc9752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 1 Nov 2018 16:03:40 +0800 Subject: [PATCH 24/42] fixed issue #1081 , support mariadb rotate_event support checksum --- .../parse/inbound/mysql/MysqlConnection.java | 68 +++++++++++++++++-- .../mysql/MysqlMultiStageCoprocessor.java | 16 ++++- .../canal/parse/DirectLogFetcherTest.java | 48 +++++++++++++ 3 files changed, 123 insertions(+), 9 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 4348dc29..e3056cb9 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -37,21 +37,23 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; public class MysqlConnection implements ErosaConnection { - private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class); + private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class); private MysqlConnector connector; private long slaveId; - private Charset charset = Charset.forName("UTF-8"); + private Charset charset = Charset.forName("UTF-8"); private BinlogFormat binlogFormat; private BinlogImage binlogImage; // tsdb releated private AuthenticationInfo authInfo; - protected int connTimeout = 5 * 1000; // 5秒 - protected int soTimeout = 60 * 60 * 1000; // 1小时 + protected int connTimeout = 5 * 1000; // 5秒 + protected int soTimeout = 60 * 60 * 1000; // 1小时 + private int binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF; // dump binlog bytes, 暂不包括meta与TSDB private AtomicLong receivedBinlogBytes; @@ -118,7 +120,7 @@ public class MysqlConnection implements ErosaConnection { */ public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { updateSettings(); - + loadBinlogChecksum(); sendBinlogDump(binlogfilename, binlogPosition); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); fetcher.start(connector.getChannel()); @@ -128,6 +130,7 @@ public class MysqlConnection implements ErosaConnection { decoder.handle(LogEvent.QUERY_EVENT); decoder.handle(LogEvent.XID_EVENT); LogContext context = new LogContext(); + context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum)); while (fetcher.fetch()) { accumulateReceivedBytes(fetcher.limit()); LogEvent event = null; @@ -145,12 +148,14 @@ public class MysqlConnection implements ErosaConnection { public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { updateSettings(); + loadBinlogChecksum(); sendRegisterSlave(); sendBinlogDump(binlogfilename, binlogPosition); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); fetcher.start(connector.getChannel()); LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); LogContext context = new LogContext(); + context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum)); while (fetcher.fetch()) { accumulateReceivedBytes(fetcher.limit()); LogEvent event = null; @@ -173,6 +178,7 @@ public class MysqlConnection implements ErosaConnection { @Override public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException { updateSettings(); + loadBinlogChecksum(); sendBinlogDumpGTID(gtidSet); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); @@ -180,6 +186,7 @@ public class MysqlConnection implements ErosaConnection { fetcher.start(connector.getChannel()); LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); LogContext context = new LogContext(); + context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum)); // fix bug: #890 将gtid传输至context中，供decode使用 context.setGtidSet(gtidSet); while (fetcher.fetch()) { @@ -207,9 +214,11 @@ public class MysqlConnection implements ErosaConnection { @Override public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException { updateSettings(); + loadBinlogChecksum(); sendRegisterSlave(); sendBinlogDump(binlogfilename, binlogPosition); ((MysqlMultiStageCoprocessor) coprocessor).setConnection(this); + ((MysqlMultiStageCoprocessor) coprocessor).setBinlogChecksum(binlogChecksum); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); try { fetcher.start(connector.getChannel()); @@ -234,9 +243,10 @@ public class MysqlConnection implements ErosaConnection { @Override public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException { updateSettings(); + loadBinlogChecksum(); sendBinlogDumpGTID(gtidSet); - ((MysqlMultiStageCoprocessor) coprocessor).setConnection(this); + ((MysqlMultiStageCoprocessor) coprocessor).setBinlogChecksum(binlogChecksum); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); try { fetcher.start(connector.getChannel()); @@ -482,6 +492,52 @@ public class MysqlConnection implements ErosaConnection { } } + /** + * 获取主库checksum信息 + * + * 
+     * mariadb区别于mysql会在binlog的第一个事件Rotate_Event里也会采用checksum逻辑,而mysql是在第二个binlog事件之后才感知是否需要处理checksum
+     * 导致maraidb只要是开启checksum就会出现binlog文件名解析乱码
+     * fixed issue : https://github.com/alibaba/canal/issues/1081
+     *
+ */ + private void loadBinlogChecksum() { + if (checkMariaDB()) { + ResultSetPacket rs = null; + try { + rs = query("select @@global.binlog_checksum"); + } catch (IOException e) { + throw new CanalParseException(e); + } + + List columnValues = rs.getFieldValues(); + if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) { + binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32; + } else { + binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF; + } + } + } + + /** + * 获取是否为mariadb + */ + private boolean checkMariaDB() { + ResultSetPacket rs = null; + try { + rs = query("SELECT @@version"); + } catch (IOException e) { + throw new CanalParseException(e); + } + + List columnValues = rs.getFieldValues(); + if (columnValues != null && columnValues.size() >= 1) { + return StringUtils.containsIgnoreCase(columnValues.get(0), "MariaDB"); + } + + return false; + } + private void accumulateReceivedBytes(long x) { if (receivedBinlogBytes != null) { receivedBinlogBytes.addAndGet(x); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 3db8ecc4..8b682f5d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -33,6 +33,7 @@ import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; @@ -69,6 +70,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement private WorkerPool workerPool; private BatchEventProcessor simpleParserStage; private BatchEventProcessor sinkStoreStage; + private LogContext logContext; public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert, EventTransactionBuffer transactionBuffer, String destination){ @@ -95,9 +97,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement SequenceBarrier sequenceBarrier = disruptorMsgBuffer.newBarrier(); ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler(); // stage 2 + this.logContext = new LogContext(); simpleParserStage = new BatchEventProcessor(disruptorMsgBuffer, sequenceBarrier, - new SimpleParserStage()); + new SimpleParserStage(logContext)); simpleParserStage.setExceptionHandler(exceptionHandler); disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence()); @@ -128,6 +131,12 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement workerPool.start(parserExecutor); } + public void setBinlogChecksum(int binlogChecksum) { + if (binlogChecksum != LogEvent.BINLOG_CHECKSUM_ALG_OFF) { + logContext.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum)); + } + } + @Override public void stop() { // fix bug #968，对于pool与 @@ -239,9 +248,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement private LogDecoder decoder; private LogContext context; - public SimpleParserStage(){ + public SimpleParserStage(LogContext context){ decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); - context = new LogContext(); + this.context = context; if (gtidSet != null) { context.setGtidSet(gtidSet); } @@ -468,4 +477,5 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement public void setGtidSet(GTIDSet gtidSet) { this.gtidSet = gtidSet; } + } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java index aa48f9a3..2cb6ed8b 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.parse; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.List; import org.apache.commons.lang.StringUtils; import org.junit.Assert; @@ -10,21 +11,27 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector; +import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor; import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor; import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; +import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; public class DirectLogFetcherTest { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + private boolean isMariaDB; + private int binlogChecksum; @Test public void testSimple() { @@ -33,6 +40,7 @@ public class DirectLogFetcherTest { MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxx", "xxxx"); connector.connect(); updateSettings(connector); + loadBinlogChecksum(connector); sendRegisterSlave(connector, 3); sendBinlogDump(connector, "mysql-bin.000001", 4L, 3); @@ -40,6 +48,8 @@ public class DirectLogFetcherTest { LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); LogContext context = new LogContext(); + context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum)); + while (fetcher.fetch()) { LogEvent event = null; event = decoder.decode(fetcher, context); @@ -188,6 +198,44 @@ public class DirectLogFetcherTest { } } + private void loadBinlogChecksum(MysqlConnector connector) { + checkMariaDB(connector); + if (isMariaDB) { + ResultSetPacket rs = null; + try { + rs = query("select @@global.binlog_checksum", connector); + } catch (IOException e) { + throw new CanalParseException(e); + } + + List columnValues = rs.getFieldValues(); + if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) { + binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32; + } else { + binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF; + } + } + } + + private void checkMariaDB(MysqlConnector connector) { + ResultSetPacket rs = null; + try { + rs = query("SELECT @@version", connector); + } catch (IOException e) { + throw new CanalParseException(e); + } + + List columnValues = rs.getFieldValues(); + if (columnValues != null && columnValues.size() >= 1) { + isMariaDB = StringUtils.containsIgnoreCase(columnValues.get(0), "MariaDB"); + } + } + + public ResultSetPacket query(String cmd, MysqlConnector connector) throws IOException { + MysqlQueryExecutor exector = new MysqlQueryExecutor(connector); + return exector.query(cmd); + } + public void update(String cmd, MysqlConnector connector) throws IOException { MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector); exector.update(cmd); From 96b9484f701f7a54534b6b10abb9136e3c61c5ab Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 16:37:47 +0800 Subject: [PATCH 25/42] modify --- .../adapter/es/service/ESSyncService.java | 143 +++++++++++++++++- .../client/adapter/es/support/ESTemplate.java | 53 ++++++- .../es/test/sync/UserSyncJoinOneTest.java | 26 ++++ .../es/test/sync/UserSyncSingleTest.java | 29 ++++ .../loader/CanalAdapterKafkaWorker.java | 6 +- .../src/main/resources/application.yml | 10 +- .../adapter/logger/LoggerAdapterExample.java | 2 +- 7 files changed, 252 insertions(+), 17 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index f34e38e1..608fde0e 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -11,13 +11,13 @@ import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.fastjson.JSON; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; @@ -35,7 +35,7 @@ public class ESSyncService { public void sync(Dml dml) { if (logger.isDebugEnabled()) { - logger.debug("DML: {}", JSON.toJSONString(dml)); + logger.debug("DML: {}", dml.toString()); } long begin = System.currentTimeMillis(); String database = dml.getDatabase(); @@ -103,7 +103,55 @@ public class ESSyncService { } private void update(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + List> oldList = dml.getOld(); + if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + int i = 0; + for (Map data : dataList) { + Map old = oldList.get(i); + if (data == null || data.isEmpty() || old == null || old.isEmpty()) { + continue; + } + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + // ------是主表 查询sql来更新------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + ESMapping mapping = config.getEsMapping(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName); + boolean idFieldSimple = true; + if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) { + idFieldSimple = false; + } + + boolean allUpdateFieldSimple = true; + out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allUpdateFieldSimple = false; + break out; + } + } + } + } + // 判断主键和所更新的字段是否全为简单字段 + if (idFieldSimple && allUpdateFieldSimple) { + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + mainTableUpdate(config, dml, data, old); + } + } + } + + i++; + } } /** @@ -123,8 +171,8 @@ public class ESSyncService { continue; } - // ------是否单表 & 所有字段都为简单字段------ if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ singleTableSimpleFiledInsert(config, dml, data); } else { // ------是主表 查询sql来插入------ @@ -210,7 +258,7 @@ public class ESSyncService { sql = ESSyncUtil.appendCondition(sql, condition); DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); if (logger.isTraceEnabled()) { - logger.trace("Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + logger.trace("Main table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", config.getDestination(), dml.getTable(), mapping.get_index(), @@ -224,7 +272,7 @@ public class ESSyncService { if (logger.isTraceEnabled()) { logger.trace( - "Single table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + "Main table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), @@ -233,7 +281,7 @@ public class ESSyncService { boolean result = esTemplate.insert(config, idVal, esFieldData); if (!result) { logger.error( - "Single table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + "Main table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), @@ -486,4 +534,87 @@ public class ESSyncService { return 0; }); } + + /** + * 单表简单字段update + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行data数据 + * @param old 单行old数据 + */ + private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map data, + Map old) { + ESMapping mapping = config.getEsMapping(); + Map esFieldData = new LinkedHashMap<>(); + + Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace("Single table update ot es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.update(config, idVal, esFieldData); + if (!result) { + logger.error("Single table update to es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + + /** + * 主表(单表)复杂字段update + * + * @param config es配置 + * @param dml dml信息 + * @param data 单行dml数据 + */ + private void mainTableUpdate(ESSyncConfig config, Dml dml, Map data, Map old) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table update ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table update ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.update(config, idVal, esFieldData); + if (!result) { + logger.error( + "Main table update to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index e81935ab..c075eb02 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -34,6 +34,7 @@ import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; public class ESTemplate { @@ -143,7 +144,7 @@ public class ESTemplate { } private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, - int counter) { + int counter) { if (CollectionUtils.isEmpty(esFieldData)) { return true; } @@ -307,6 +308,28 @@ public class ESTemplate { return resultIdVal; } + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + } + + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (dmlOld.get(columnItem.getColumnName()) != null + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); + break; + } + } + } + return resultIdVal; + } + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { String esType = getEsType(mapping, fieldName); Object value = dmlData.get(columnName); @@ -347,6 +370,34 @@ public class ESTemplate { return resultIdVal; } + /** + * 将dml的data, old转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + } + + if (dmlOld.get(columnName) != null && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); + } + } + return resultIdVal; + } + /** * es 字段类型本地缓存 */ diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java index f2f9f4e3..2406f301 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java @@ -51,6 +51,32 @@ public class UserSyncJoinOneTest { Assert.assertEquals("Eric_", response.getSource().get("_name")); } + @Test + public void updateTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric2"); + dml.setOld(oldList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric_", response.getSource().get("_name")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java index 16a25bea..a772d754 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java @@ -54,6 +54,35 @@ public class UserSyncSingleTest { Assert.assertEquals("Eric", response.getSource().get("_name")); } + /** + * 单表更新 + */ + @Test + public void updateTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric2"); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("name", "Eric"); + dml.setOld(oldList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("Eric2", response.getSource().get("_name")); + } + @Test public void ttt() { SearchResponse response = esAdapter.getTransportClient() diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java index b8de5269..2e861642 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -1,16 +1,14 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.util.List; -import java.util.concurrent.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; -import org.apache.kafka.clients.consumer.CommitFailedException; import org.apache.kafka.common.errors.WakeupException; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; /** * kafka对应的client适配器工作线程 diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 3e0e0dc4..9fe6aa2d 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -27,11 +27,11 @@ canal.conf: - instance: example groups: - outAdapters: - - name: es - hosts: 127.0.0.1:9300 - properties: - cluster.name: elasticsearch -# - name: logger +# - name: es +# hosts: 127.0.0.1:9300 +# properties: +# cluster.name: elasticsearch + - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index c7b7978d..8f95b212 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -27,7 +27,7 @@ public class LoggerAdapterExample implements OuterAdapter { @Override public void sync(Dml dml) { - logger.info("DML: ", dml.toString()); + logger.info("DML: {}", dml.toString()); } @Override From 332f759f96f7ff7a6c1b771fa94a493c979b6eaa Mon Sep 17 00:00:00 2001 From: wuwo Date: Thu, 1 Nov 2018 17:38:18 +0800 Subject: [PATCH 26/42] filter dml before apply to tsdb --- .../canal/parse/inbound/mysql/dbsync/LogEventConvert.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 74db38fa..0b88af14 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -260,7 +260,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return null; } - if (!isSeek) { + boolean isDml = (type == EventType.INSERT || type == EventType.UPDATE || type == EventType.DELETE); + + if (!isSeek && !isDml) { // 使用新的表结构元数据管理方式 EntryPosition position = createPosition(event.getHeader()); tableMetaCache.apply(position, event.getDbName(), queryString, null); @@ -268,8 +270,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar Header header = createHeader(event.getHeader(), schemaName, tableName, type); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); - if (type != EventType.QUERY && type != EventType.INSERT && type != EventType.UPDATE - && type != EventType.DELETE) { + if (type != EventType.QUERY && !isDml) { rowChangeBuider.setIsDdl(true); } rowChangeBuider.setSql(queryString); From 81f403a9f2820f24cbc35f4513f14ac26283de07 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 1 Nov 2018 21:07:45 +0800 Subject: [PATCH 27/42] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/es/service/ESSyncService.java | 143 +++++++++++++++--- .../es/test/sync/RoleSyncJoinOne2Test.java | 27 ++++ .../es/test/sync/RoleSyncJoinOneTest.java | 27 ++++ .../es/test/sync/RoleSyncJoinSub2Test.java | 28 ++++ .../es/test/sync/RoleSyncJoinSubTest.java | 28 ++++ .../src/main/resources/application-sample.yml | 20 ++- .../src/main/resources/application.yml | 4 +- 7 files changed, 248 insertions(+), 29 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 608fde0e..3c50d5c6 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -15,9 +15,9 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; @@ -120,7 +120,7 @@ public class ESSyncService { // ------单表 & 所有字段都为简单字段------ singleTableSimpleFiledUpdate(config, dml, data, old); } else { - // ------是主表 查询sql来更新------ + // ------主表 查询sql来更新------ if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { ESMapping mapping = config.getEsMapping(); String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); @@ -148,6 +148,50 @@ public class ESSyncService { mainTableUpdate(config, dml, data, old); } } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, old, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + jonTableWholeSqlOperation(config, dml, data, old, tableItem); + } + } } i++; @@ -201,14 +245,23 @@ public class ESSyncService { // 不是子查询 if (!tableItem.isSubQuery()) { // ------关联表简单字段插入------ - joinTableSimpleFieldOperation(config, dml, data, tableItem); + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); } else { // ------关联子表简单字段插入------ - subTableSimpleFieldOperation(config, dml, data, tableItem); + subTableSimpleFieldOperation(config, dml, data, null, tableItem); } } else { // ------关联子表复杂字段插入 执行全sql更新es------ - jonTableWholeSqlOperation(config, dml, data, tableItem); + jonTableWholeSqlOperation(config, dml, data, null, tableItem); } } } @@ -304,14 +357,8 @@ public class ESSyncService { * @param tableItem 当前表配置 */ private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, - TableItem tableItem) { + TableItem tableItem, Map esFieldData) { ESMapping mapping = config.getEsMapping(); - Map esFieldData = new LinkedHashMap<>(); - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object value = esTemplate - .getValFromData(mapping, data, fieldItem.getFieldName(), fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), value); - } BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { @@ -353,10 +400,11 @@ public class ESSyncService { * @param config es配置 * @param dml dml信息 * @param data 单行dml数据 + * @param old 单行old数据 * @param tableItem 当前表配置 */ private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map data, - TableItem tableItem) { + Map old, TableItem tableItem) { ESMapping mapping = config.getEsMapping(); StringBuilder sql = new StringBuilder( "SELECT * FROM (" + tableItem.getSubQuerySql() + ") " + tableItem.getAlias() + " WHERE "); @@ -396,10 +444,31 @@ public class ESSyncService { try { while (rs.next()) { Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object val = esTemplate - .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), val); + if (old != null) { + out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) { + for (ColumnItem columnItem0 : fieldItem.getColumnItems()) { + if (fieldItem1.getFieldName().equals(columnItem0.getColumnName())) + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), val); + break out; + } + } + } + } + } else { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), val); + } } BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); @@ -451,7 +520,7 @@ public class ESSyncService { * @param tableItem 当前表配置 */ private void jonTableWholeSqlOperation(ESSyncConfig config, Dml dml, Map data, - TableItem tableItem) { + Map old, TableItem tableItem) { ESMapping mapping = config.getEsMapping(); StringBuilder sql = new StringBuilder(mapping.getSql() + " WHERE "); @@ -491,9 +560,43 @@ public class ESSyncService { while (rs.next()) { Map esFieldData = new LinkedHashMap<>(); for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object val = esTemplate - .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); - esFieldData.put(fieldItem.getFieldName(), val); + if (old != null) { + // 从表子查询 + out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) { + for (ColumnItem columnItem0 : fieldItem.getColumnItems()) { + if (fieldItem1.getFieldName().equals(columnItem0.getColumnName())) + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + break out; + } + } + } + } + // 从表非子查询 + for (FieldItem fieldItem1 : tableItem.getRelationSelectFieldItems()) { + if (fieldItem1.equals(fieldItem)) { + for (ColumnItem columnItem : fieldItem1.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + Object val = esTemplate.getValFromRS(mapping, + rs, + fieldItem.getFieldName(), + fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + break; + } + } + } + } + } else { + Object val = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + esFieldData.put(fieldItem.getFieldName(), val); + } } BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java index 577ec758..42e119db 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java @@ -49,6 +49,33 @@ public class RoleSyncJoinOne2Test { Assert.assertEquals("admin2_", response.getSource().get("_role_name")); } + @Test + public void updateTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin3"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_name", "admin"); + dml.setOld(oldList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin3_", response.getSource().get("_role_name")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java index dde3351d..4bfa2809 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java @@ -49,6 +49,33 @@ public class RoleSyncJoinOneTest { Assert.assertEquals("admin", response.getSource().get("_role_name")); } + @Test + public void updateTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin2"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_name", "admin"); + dml.setOld(oldList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin2", response.getSource().get("_role_name")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java index 4d67139b..f67e0b33 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java @@ -50,6 +50,34 @@ public class RoleSyncJoinSub2Test { Assert.assertEquals("a;b_", response.getSource().get("_labels")); } + @Test + public void updateTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id",1L); + data.put("label", "aa"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("label", "a"); + dml.setOld(oldList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("aa;b_", response.getSource().get("_labels")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java index c56d1d3c..5cc555a8 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java @@ -50,6 +50,34 @@ public class RoleSyncJoinSubTest { Assert.assertEquals("a;b", response.getSource().get("_labels")); } + @Test + public void updateTest02() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id",1L); + data.put("label", "aa"); + dml.setData(dataList); + + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("label", "a"); + dml.setOld(oldList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("aa;b", response.getSource().get("_labels")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/launcher/src/main/resources/application-sample.yml b/client-adapter/launcher/src/main/resources/application-sample.yml index 706e4088..911f58c3 100644 --- a/client-adapter/launcher/src/main/resources/application-sample.yml +++ b/client-adapter/launcher/src/main/resources/application-sample.yml @@ -18,6 +18,10 @@ canal.conf: - instance: example groups: - outAdapters: +# - name: es +# hosts: 127.0.0.1:9300 +# properties: +# cluster.name: elasticsearch - name: logger # - name: hbase # properties: @@ -39,11 +43,11 @@ canal.conf: # outAdapters: # - name: logger # -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: -# - hbase/mytest_person2.yml +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: + - hbase/mytest_person2.yml diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 7fe7b191..312909eb 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -7,4 +7,6 @@ spring: jackson: date-format: yyyy-MM-dd HH:mm:ss time-zone: GMT+8 - default-property-inclusion: non_null \ No newline at end of file + default-property-inclusion: non_null + profiles: + active: sample \ No newline at end of file From a9727cf267c2a0f7e0abe80b68285f86ffa3e45e Mon Sep 17 00:00:00 2001 From: machey Date: Thu, 1 Nov 2018 23:43:53 +0800 Subject: [PATCH 28/42] =?UTF-8?q?release=E5=8C=85=E4=B8=8D=E6=89=93?= =?UTF-8?q?=E8=BF=9Bapplication.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/launcher/pom.xml | 18 +++++-- .../src/main/resources/application-sample.yml | 49 ------------------- .../src/main/resources/application.yml | 41 +++++++++++++++- 3 files changed, 54 insertions(+), 54 deletions(-) delete mode 100644 client-adapter/launcher/src/main/resources/application-sample.yml diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index 1a79ed21..f70254b0 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -1,5 +1,6 @@ - + canal.client-adapter com.alibaba.otter @@ -134,12 +135,13 @@ - + - - + + @@ -201,6 +203,14 @@ + + + src/main/resources + + application.yml + + + diff --git a/client-adapter/launcher/src/main/resources/application-sample.yml b/client-adapter/launcher/src/main/resources/application-sample.yml deleted file mode 100644 index 706e4088..00000000 --- a/client-adapter/launcher/src/main/resources/application-sample.yml +++ /dev/null @@ -1,49 +0,0 @@ -server: - port: 8081 -logging: - level: - com.alibaba.otter.canal.client.adapter.hbase: DEBUG -spring: - jackson: - date-format: yyyy-MM-dd HH:mm:ss - time-zone: GMT+8 - default-property-inclusion: non_null - -canal.conf: - canalServerHost: 127.0.0.1:11111 -# zookeeperHosts: slave1:2181 -# bootstrapServers: slave1:6667 #or rocketmq -# flatMessage: true - canalInstances: - - instance: example - groups: - - outAdapters: - - name: logger -# - name: hbase -# properties: -# hbase.zookeeper.quorum: 127.0.0.1 -# hbase.zookeeper.property.clientPort: 2181 -# zookeeper.znode.parent: /hbase -# mqTopics: -# - mqMode: kafka -# topic: example -# groups: -# - groupId: g2 -# outAdapters: -# - name: logger -# mqTopics: -# - mqMode: rocketmq -# topic: example -# groups: -# - groupId: g2 -# outAdapters: -# - name: logger -# -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: -# - hbase/mytest_person2.yml diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 7fe7b191..ae3648b7 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -7,4 +7,43 @@ spring: jackson: date-format: yyyy-MM-dd HH:mm:ss time-zone: GMT+8 - default-property-inclusion: non_null \ No newline at end of file + default-property-inclusion: non_null + +#canal.conf: +# canalServerHost: 127.0.0.1:11111 +# zookeeperHosts: slave1:2181 +# bootstrapServers: slave1:6667 #or rocketmq +# flatMessage: true +# canalInstances: +# - instance: example +# groups: +# - outAdapters: +# - name: logger +# - name: hbase +# properties: +# hbase.zookeeper.quorum: 127.0.0.1 +# hbase.zookeeper.property.clientPort: 2181 +# zookeeper.znode.parent: /hbase +# mqTopics: +# - mqMode: kafka +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger +# mqTopics: +# - mqMode: rocketmq +# topic: example +# groups: +# - groupId: g2 +# outAdapters: +# - name: logger +# +#adapter.conf: +# datasourceConfigs: +# defaultDS: +# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true +# username: root +# password: 121212 +# adapterConfigs: +# - hbase/mytest_person2.yml From 9473f4e8fdd1c677ee30bead6b8abfd25c0eb14e Mon Sep 17 00:00:00 2001 From: jiacheo Date: Fri, 2 Nov 2018 01:48:17 +0800 Subject: [PATCH 29/42] Update CanalMessageDeserializer.java fix lazyParseEntry 'raw' wrong value. --- .../alibaba/otter/canal/client/CanalMessageDeserializer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java index b2206890..9d24a23b 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java @@ -32,10 +32,12 @@ public class CanalMessageDeserializer { if (lazyParseEntry) { // byteString result.setRawEntries(messages.getMessagesList()); + result.setRaw(true); } else { for (ByteString byteString : messages.getMessagesList()) { result.addEntry(CanalEntry.Entry.parseFrom(byteString)); } + result.setRaw(false); } return result; } From 5b9ada1f8c10dc1f5d8779e686f4f9ade4b4e9a3 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 2 Nov 2018 09:51:30 +0800 Subject: [PATCH 30/42] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/es/service/ESSyncService.java | 198 ++++++++---------- .../client/adapter/es/support/ESSyncUtil.java | 13 +- .../canal/client/adapter/es/support/Util.java | 18 -- .../es/test/sync/RoleSyncJoinSub2Test.java | 2 +- 4 files changed, 96 insertions(+), 135 deletions(-) delete mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index 3c50d5c6..c2495416 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -102,6 +102,82 @@ public class ESSyncService { } + /** + * 插入操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void insert(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledInsert(config, dml, data); + } else { + // ------是主表 查询sql来插入------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + mainTableInsert(config, dml, data); + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段插入------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段插入------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段插入 执行全sql更新es------ + jonTableWholeSqlOperation(config, dml, data, null, tableItem); + } + } + } + } + } + + /** + * 更新操作dml + * + * @param config es配置 + * @param dml dml数据 + */ private void update(ESSyncConfig config, Dml dml) { List> dataList = dml.getData(); List> oldList = dml.getOld(); @@ -198,76 +274,6 @@ public class ESSyncService { } } - /** - * 插入dml操作 - * - * @param config es配置 - * @param dml dml数据 - */ - private void insert(ESSyncConfig config, Dml dml) { - List> dataList = dml.getData(); - if (dataList == null || dataList.isEmpty()) { - return; - } - SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); - for (Map data : dataList) { - if (data == null || data.isEmpty()) { - continue; - } - - if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { - // ------单表 & 所有字段都为简单字段------ - singleTableSimpleFiledInsert(config, dml, data); - } else { - // ------是主表 查询sql来插入------ - if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { - mainTableInsert(config, dml, data); - } - - // 从表的操作 - for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { - if (tableItem.isMain()) { - continue; - } - if (!tableItem.getTableName().equals(dml.getTable())) { - continue; - } - // 关联条件出现在主表查询条件是否为简单字段 - boolean allFieldsSimple = true; - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { - allFieldsSimple = false; - break; - } - } - // 所有查询字段均为简单字段 - if (allFieldsSimple) { - // 不是子查询 - if (!tableItem.isSubQuery()) { - // ------关联表简单字段插入------ - Map esFieldData = new LinkedHashMap<>(); - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object value = esTemplate.getValFromData(config.getEsMapping(), - data, - fieldItem.getFieldName(), - fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), value); - } - - joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); - } else { - // ------关联子表简单字段插入------ - subTableSimpleFieldOperation(config, dml, data, null, tableItem); - } - } else { - // ------关联子表复杂字段插入 执行全sql更新es------ - jonTableWholeSqlOperation(config, dml, data, null, tableItem); - } - } - } - } - } - /** * 单表简单字段insert * @@ -412,23 +418,7 @@ public class ESSyncService { for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { String columnName = fkFieldItem.getColumn().getColumnName(); Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); - if (value instanceof String) { - sql.append(tableItem.getAlias()) - .append(".") - .append(columnName) - .append("='") - .append(value) - .append("' ") - .append(" AND "); - } else { - sql.append(tableItem.getAlias()) - .append(".") - .append(columnName) - .append("=") - .append(value) - .append(" ") - .append(" AND "); - } + ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName); } int len = sql.length(); sql.delete(len - 5, len); @@ -527,23 +517,7 @@ public class ESSyncService { for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) { String columnName = fkFieldItem.getColumn().getColumnName(); Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName); - if (value instanceof String) { - sql.append(tableItem.getAlias()) - .append(".") - .append(columnName) - .append("='") - .append(value) - .append("' ") - .append(" AND "); - } else { - sql.append(tableItem.getAlias()) - .append(".") - .append(columnName) - .append("=") - .append(value) - .append(" ") - .append(" AND "); - } + ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName); } int len = sql.length(); sql.delete(len - 5, len); @@ -602,16 +576,14 @@ public class ESSyncService { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { for (FieldItem fieldItem : entry.getValue()) { - if (fieldItem.getColumnItems().size() == 1) { - Object value = esTemplate - .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); - String fieldName = fieldItem.getFieldName(); - // 判断是否是主键 - if (fieldName.equals(mapping.get_id())) { - fieldName = "_id"; - } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + Object value = esTemplate + .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName()); + String fieldName = fieldItem.getFieldName(); + // 判断是否是主键 + if (fieldName.equals(mapping.get_id())) { + fieldName = "_id"; } + queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index 88f7b496..71331873 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -19,7 +19,6 @@ import com.alibaba.fastjson.JSON; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; public class ESSyncUtil { @@ -161,7 +160,7 @@ public class ESSyncUtil { } if (!((String) val).contains(",")) { - logger.error("es type is geo_point, source value not contains ',' spit"); + logger.error("es type is geo_point, source value not contains ',' separator"); return val; } @@ -225,7 +224,7 @@ public class ESSyncUtil { // 拼接condition StringBuilder condition = new StringBuilder(" "); for (ColumnItem idColumn : idColumns) { - Object idVal = data.get(Util.cleanColumn(idColumn.getColumnName())); + Object idVal = data.get(idColumn.getColumnName()); if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append("."); condition.append(idColumn.getColumnName()).append("="); if (idVal instanceof String) { @@ -246,6 +245,14 @@ public class ESSyncUtil { return sql + " WHERE " + condition + " "; } + public static void appendCondition(StringBuilder sql, Object value, String owner, String columnName) { + if (value instanceof String) { + sql.append(owner).append(".").append(columnName).append("='").append(value).append("' AND "); + } else { + sql.append(owner).append(".").append(columnName).append("=").append(value).append(" AND "); + } + } + /** * 执行查询sql */ diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java deleted file mode 100644 index a801b9e6..00000000 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/Util.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.alibaba.otter.canal.client.adapter.es.support; - -public class Util { - public static String cleanColumn(String column) { - if (column == null) { - return null; - } - if (column.contains("`")) { - column = column.replaceAll("`", ""); - } - - if (column.contains("'")) { - column = column.replaceAll("'", ""); - } - - return column; - } -} diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java index f67e0b33..469894d3 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java @@ -69,7 +69,7 @@ public class RoleSyncJoinSub2Test { List> oldList = new ArrayList<>(); Map old = new LinkedHashMap<>(); oldList.add(old); - old.put("label", "a"); + old.put("label", "v"); dml.setOld(oldList); esAdapter.getEsSyncService().sync(dml); From 5cc2f8f9f7314d48934343ba1e988b83c2868cac Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 2 Nov 2018 11:07:41 +0800 Subject: [PATCH 31/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0file=E7=9A=84path?= =?UTF-8?q?=E8=AF=BB=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/support/ExtensionLoader.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java index 5b5c1bbb..24d43a0f 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java @@ -292,10 +292,13 @@ public class ExtensionLoader { private String getJarDirectoryPath() { URL url = Thread.currentThread().getContextClassLoader().getResource(""); - if (url == null) { - throw new IllegalStateException("failed to get class loader resource"); + String dirtyPath; + if (url != null) { + dirtyPath = url.toString(); + } else { + File file = new File(""); + dirtyPath = file.getAbsolutePath(); } - String dirtyPath = url.toString(); String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes // file:/ and // everything From 8597d9e926c9782c1e5def45947995611cf20b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 2 Nov 2018 12:51:36 +0800 Subject: [PATCH 32/42] fixed more info --- .../parse/driver/mysql/socket/BioSocketChannel.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java index 8de2acfd..dfa2783f 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java @@ -88,8 +88,9 @@ public class BioSocketChannel implements SocketChannel { } } if (remain > 0 && accTimeout >= timeout) { - throw new SocketTimeoutException("Timeout occurred, failed to read " + readSize + " bytes in " + timeout - + " milliseconds."); + throw new SocketTimeoutException("Timeout occurred, failed to read total " + readSize + " bytes in " + + timeout + " milliseconds, actual read only " + (readSize - remain) + + " bytes"); } return data; } @@ -120,8 +121,8 @@ public class BioSocketChannel implements SocketChannel { } if (n < len && accTimeout >= timeout) { - throw new SocketTimeoutException("Timeout occurred, failed to read " + len + " bytes in " + timeout - + " milliseconds."); + throw new SocketTimeoutException("Timeout occurred, failed to read total " + len + " bytes in " + timeout + + " milliseconds, actual read only " + n + " bytes"); } } From 391e040c6a448bed08f510650b18ebb28024f964 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 2 Nov 2018 17:19:52 +0800 Subject: [PATCH 33/42] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E5=AE=8C=E6=88=90=20=E5=90=8E=E7=BB=AD:=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=20=E5=88=A0=E9=99=A4=E5=A4=96=E9=94=AE=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5,=20=E4=BF=AE=E6=94=B9=E4=B8=BB=E9=94=AE=E7=9A=84?= =?UTF-8?q?=E6=83=85=E5=86=B5=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/es/config/ESSyncConfigLoader.java | 5 +- .../adapter/es/service/ESSyncService.java | 136 +++++++++++++++++- .../client/adapter/es/support/ESTemplate.java | 35 +++-- .../src/main/resources/es/mytest_user.yml | 17 +-- ...b2Test.java => LabelSyncJoinSub2Test.java} | 25 +++- ...SubTest.java => LabelSyncJoinSubTest.java} | 27 +++- .../es/test/sync/RoleSyncJoinOneTest.java | 22 +++ .../es/test/sync/UserSyncSingleTest.java | 37 +++-- .../resources/es/mytest_user_join_sub.yml | 2 +- .../resources/es/mytest_user_join_sub2.yml | 2 +- .../hbase/config/MappingConfigLoader.java | 3 + .../src/main/resources/application.yml | 36 +++-- 12 files changed, 285 insertions(+), 62 deletions(-) rename client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/{RoleSyncJoinSub2Test.java => LabelSyncJoinSub2Test.java} (77%) rename client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/{RoleSyncJoinSubTest.java => LabelSyncJoinSubTest.java} (75%) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index 7a54cccd..7fef12c0 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -38,6 +38,9 @@ public class ESSyncConfigLoader { public static synchronized void load() { logger.info("## Start loading mapping config ... "); Collection configs = AdapterConfigs.get("es"); + if (configs == null) { + return; + } for (String c : configs) { if (c == null) { continue; @@ -67,7 +70,7 @@ public class ESSyncConfigLoader { } Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); Matcher matcher = pattern.matcher(dataSource.getUrl()); - if (!matcher.find()){ + if (!matcher.find()) { throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); } String schema = matcher.group(2); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index c2495416..cbab4c64 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -99,7 +99,88 @@ public class ESSyncService { } private void delete(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + ESMapping mapping = config.getEsMapping(); + + // ------是主表------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping); + // 主键为简单字段 + if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) { + Object idVal = esTemplate.getValFromData(mapping, + data, + idFieldItem.getFieldName(), + idFieldItem.getColumn().getColumnName()); + + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.delete(mapping, idVal); + if (!result) { + logger.error("Main table delete es index error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } else { + // ------主键带函数, 查询sql获取主键删除------ + mainTableDelete(config, dml, data); + } + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新为null------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + esFieldData.put(fieldItem.getFieldName(), null); + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + jonTableWholeSqlOperation(config, dml, data, null, tableItem); + } + } + } } /** @@ -293,7 +374,7 @@ public class ESSyncService { mapping.get_index(), idVal); } - boolean result = esTemplate.insert(config, idVal, esFieldData); + boolean result = esTemplate.insert(mapping, idVal, esFieldData); if (!result) { logger.error("Single table insert to es index error, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), @@ -337,7 +418,7 @@ public class ESSyncService { mapping.get_index(), idVal); } - boolean result = esTemplate.insert(config, idVal, esFieldData); + boolean result = esTemplate.insert(mapping, idVal, esFieldData); if (!result) { logger.error( "Main table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", @@ -354,6 +435,49 @@ public class ESSyncService { }); } + private void mainTableDelete(ESSyncConfig config, Dml dml, Map data) { + ESMapping mapping = config.getEsMapping(); + String sql = mapping.getSql(); + String condition = ESSyncUtil.pkConditionSql(mapping, data); + sql = ESSyncUtil.appendCondition(sql, condition); + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + if (logger.isTraceEnabled()) { + logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + sql.replace("\n", " ")); + } + ESSyncUtil.sqlRS(ds, sql, rs -> { + try { + while (rs.next()) { + Object idVal = esTemplate.getIdValFromRS(mapping, rs); + + if (logger.isTraceEnabled()) { + logger.trace( + "Main table delete ot es index by query sql, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + boolean result = esTemplate.delete(mapping, idVal); + if (!result) { + logger.error( + "Main table delete to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", + config.getDestination(), + dml.getTable(), + mapping.get_index(), + idVal); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + } + /** * 关联表主表简单字段operation * @@ -626,15 +750,15 @@ public class ESSyncService { Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData); if (logger.isTraceEnabled()) { - logger.trace("Single table update ot es index, destination:{}, table: {}, index: {}, id: {}", + logger.trace("Main table update ot es index, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal); } - boolean result = esTemplate.update(config, idVal, esFieldData); + boolean result = esTemplate.update(mapping, idVal, esFieldData); if (!result) { - logger.error("Single table update to es index error, destination:{}, table: {}, index: {}, id: {}", + logger.error("Main table update to es index error, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), @@ -676,7 +800,7 @@ public class ESSyncService { mapping.get_index(), idVal); } - boolean result = esTemplate.update(config, idVal, esFieldData); + boolean result = esTemplate.update(mapping, idVal, esFieldData); if (!result) { logger.error( "Main table update to es index by query sql error, destination:{}, table: {}, index: {}, id: {}", diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index c075eb02..655087e7 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; @@ -56,14 +55,13 @@ public class ESTemplate { /** * 插入数据 * - * @param esSyncConfig + * @param mapping * @param pkVal * @param esFieldData * @return */ - public boolean insert(ESSyncConfig esSyncConfig, Object pkVal, Map esFieldData) { + public boolean insert(ESMapping mapping, Object pkVal, Map esFieldData) { BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - ESMapping mapping = esSyncConfig.getEsMapping(); if (mapping.get_id() != null) { bulkRequestBuilder .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) @@ -87,14 +85,14 @@ public class ESTemplate { /** * 根据主键更新数据 * - * @param esSyncConfig + * @param mapping * @param pkVal * @param esFieldData * @return */ - public boolean update(ESSyncConfig esSyncConfig, Object pkVal, Map esFieldData) { + public boolean update(ESMapping mapping, Object pkVal, Map esFieldData) { BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - append4Update(bulkRequestBuilder, esSyncConfig, pkVal, esFieldData); + append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData); return commitBulkRequest(bulkRequestBuilder); } @@ -110,9 +108,8 @@ public class ESTemplate { return commitBulkRequest(bulkRequestBuilder); } - public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESSyncConfig esSyncConfig, Object pkVal, + public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, Map esFieldData) { - ESMapping mapping = esSyncConfig.getEsMapping(); if (mapping.get_id() != null) { bulkRequestBuilder .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) @@ -223,13 +220,12 @@ public class ESTemplate { /** * 通过主键删除数据 * - * @param esSyncConfig + * @param mapping * @param pkVal * @return */ - public boolean delete(ESSyncConfig esSyncConfig, Object pkVal) { + public boolean delete(ESMapping mapping, Object pkVal) { BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - ESMapping mapping = esSyncConfig.getEsMapping(); if (mapping.get_id() != null) { bulkRequestBuilder .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); @@ -308,6 +304,21 @@ public class ESTemplate { return resultIdVal; } + public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + break; + } + } + return resultIdVal; + } + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, Map esFieldData) throws SQLException { SchemaItem schemaItem = mapping.getSchemaItem(); diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 7a407f71..81dd7b3d 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -3,15 +3,12 @@ destination: example esMapping: _index: mytest_user _type: _doc - _id: id + _id: _id # pk: id -# sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a -# left join role b on a.role_id=b.id -# left join (select user_id, group_concat(label,',') as labels from user_label -# group by user_id) c on c.user_id=a.id" -# sql: "select id, name, c_time from user " -# sql: "select id, concat(name, '_') as name, c_time from user " - sql: "select a.id, a.name, a.role_id, b.role_name, a.c_time from user a - left join role b on b.id=a.role_id" + sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, + a.c_time as _c_time, c.labels as _labels from user a + left join role b on b.id=a.role_id + left join (select user_id, group_concat(label order by id desc separator ';') as labels from label + group by user_id) c on c.user_id=a.id" commitBatch: 3000 - etlCondition: "where a.c_time>='{0}' or b.c_time>='{0}' or c.c_time>='{0}'" \ No newline at end of file + etlCondition: "where a.c_time>='{0}'" \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java similarity index 77% rename from client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java rename to client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java index 469894d3..f418c1bf 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSub2Test.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java @@ -14,7 +14,7 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; -public class RoleSyncJoinSub2Test { +public class LabelSyncJoinSub2Test { private ESAdapter esAdapter; @@ -78,6 +78,29 @@ public class RoleSyncJoinSub2Test { Assert.assertEquals("aa;b_", response.getSource().get("_labels")); } + @Test + public void deleteTest03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id",1L); + data.put("label", "a"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b_", response.getSource().get("_labels")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java similarity index 75% rename from client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java rename to client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java index 5cc555a8..9c7944b0 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java @@ -14,7 +14,7 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; -public class RoleSyncJoinSubTest { +public class LabelSyncJoinSubTest { private ESAdapter esAdapter; @@ -47,7 +47,7 @@ public class RoleSyncJoinSubTest { esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("a;b", response.getSource().get("_labels")); + Assert.assertEquals("b;a", response.getSource().get("_labels")); } @Test @@ -78,6 +78,29 @@ public class RoleSyncJoinSubTest { Assert.assertEquals("aa;b", response.getSource().get("_labels")); } + @Test + public void deleteTest03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("label"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("user_id",1L); + data.put("label", "a"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("b", response.getSource().get("_labels")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java index 4bfa2809..aa383684 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java @@ -76,6 +76,28 @@ public class RoleSyncJoinOneTest { Assert.assertEquals("admin2", response.getSource().get("_role_name")); } + @Test + public void deleteTest03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("role"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_name", "admin"); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertNull(response.getSource().get("_role_name")); + } + @After public void after() { esAdapter.destroy(); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java index a772d754..98fa1d6b 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java @@ -3,9 +3,6 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync; import java.util.*; import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -83,17 +80,31 @@ public class UserSyncSingleTest { Assert.assertEquals("Eric2", response.getSource().get("_name")); } + /** + * 单表删除 + */ @Test - public void ttt() { - SearchResponse response = esAdapter.getTransportClient() - .prepareSearch("mytest_user") - .setTypes("_doc") - .setQuery(QueryBuilders.termQuery("_id", 1L)) - .setSize(100) - .get(); - for (SearchHit hit : response.getHits()) { - System.out.println(hit.getSourceAsMap().get("name")); - } + public void deleteTest03() { + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("DELETE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("name", "Eric"); + data.put("role_id", 1L); + data.put("c_time", new Date()); + + dml.setData(dataList); + + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertNull(response.getSource()); } @After diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml index 2583d40d..82c4e247 100644 --- a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml @@ -6,6 +6,6 @@ esMapping: _id: _id sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, b.labels _labels, a.c_time as _c_time from user a - left join (select user_id, group_concat(label separator ';') as labels from label + left join (select user_id, group_concat(label order by id desc separator ';') as labels from label group by user_id) b on b.user_id=a.id" commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml index 8ae06ab4..c8c22c36 100644 --- a/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml +++ b/client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml @@ -6,6 +6,6 @@ esMapping: _id: _id sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id, concat(b.labels, '_') as _labels, a.c_time as _c_time from user a - left join (select user_id, group_concat(label separator ';') as labels from label + left join (select user_id, group_concat(label order by id desc separator ';') as labels from label group by user_id) b on b.user_id=a.id" commitBatch: 3000 \ No newline at end of file diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index c0e9496a..cd5407d8 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -38,6 +38,9 @@ public class MappingConfigLoader { Map result = new LinkedHashMap<>(); Collection configs = AdapterConfigs.get("hbase"); + if (configs == null) { + return result; + } for (String c : configs) { if (c == null) { continue; diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index ae3648b7..bd70c5e0 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -3,27 +3,32 @@ server: logging: level: com.alibaba.otter.canal.client.adapter.hbase: DEBUG + com.alibaba.otter.canal.client.adapter.es: TRACE spring: jackson: date-format: yyyy-MM-dd HH:mm:ss time-zone: GMT+8 default-property-inclusion: non_null -#canal.conf: -# canalServerHost: 127.0.0.1:11111 +canal.conf: + canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 # bootstrapServers: slave1:6667 #or rocketmq # flatMessage: true -# canalInstances: -# - instance: example -# groups: -# - outAdapters: + canalInstances: + - instance: example + groups: + - outAdapters: # - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: 127.0.0.1 # hbase.zookeeper.property.clientPort: 2181 # zookeeper.znode.parent: /hbase + - name: es + hosts: 127.0.0.1:9300 + properties: + cluster.name: elasticsearch # mqTopics: # - mqMode: kafka # topic: example @@ -38,12 +43,13 @@ spring: # - groupId: g2 # outAdapters: # - name: logger -# -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: -# - hbase/mytest_person2.yml + +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: + - hbase/mytest_person2.yml + - es/mytest_user.yml From 5c8a5998d95a28270f7e1219dc26534cca1a8384 Mon Sep 17 00:00:00 2001 From: mcy Date: Mon, 5 Nov 2018 10:06:45 +0800 Subject: [PATCH 34/42] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=96=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/es/config/ESSyncConfig.java | 33 +- .../adapter/es/service/ESEtlService.java | 267 ++++++++++++ .../adapter/es/service/ESSyncService.java | 390 ++++++++++-------- .../client/adapter/es/support/ESTemplate.java | 2 +- .../src/main/resources/es/mytest_user.yml | 4 +- 5 files changed, 507 insertions(+), 189 deletions(-) create mode 100644 client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java index 440ecb6a..e327b93f 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -58,12 +58,13 @@ public class ESSyncConfig { private String pk; private String parent; private String sql; - private List skips = new ArrayList<>(); - private boolean alwaysSql = false; - private int commitBatch = 1000; + private List skips = new ArrayList<>(); + private int commitBatch = 1000; private String etlCondition; + private boolean syncByTimestamp = false; // 是否按时间戳定时同步 + private Long syncInterval; // 同步时间间隔 - private SchemaItem schemaItem; // sql解析结果模型 + private SchemaItem schemaItem; // sql解析结果模型 public String get_index() { return _index; @@ -121,14 +122,6 @@ public class ESSyncConfig { this.sql = sql; } - public boolean isAlwaysSql() { - return alwaysSql; - } - - public void setAlwaysSql(boolean alwaysSql) { - this.alwaysSql = alwaysSql; - } - public int getCommitBatch() { return commitBatch; } @@ -145,6 +138,22 @@ public class ESSyncConfig { this.etlCondition = etlCondition; } + public Long getSyncInterval() { + return syncInterval; + } + + public void setSyncInterval(Long syncInterval) { + this.syncInterval = syncInterval; + } + + public boolean isSyncByTimestamp() { + return syncByTimestamp; + } + + public void setSyncByTimestamp(boolean syncByTimestamp) { + this.syncByTimestamp = syncByTimestamp; + } + public SchemaItem getSchemaItem() { return schemaItem; } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java new file mode 100644 index 00000000..12a41a73 --- /dev/null +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java @@ -0,0 +1,267 @@ +package com.alibaba.otter.canal.client.adapter.es.service; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.sql.DataSource; + +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.rest.RestStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil; +import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.google.common.base.Joiner; + +public class ESEtlService { + + private static Logger logger = LoggerFactory.getLogger(ESEtlService.class); + + private TransportClient transportClient; + private ESTemplate esTemplate; + private ESSyncConfig config; + + public ESEtlService(TransportClient transportClient, ESSyncConfig config){ + this.transportClient = transportClient; + this.esTemplate = new ESTemplate(transportClient); + this.config = config; + } + + public EtlResult importData(List params, boolean bulk) { + EtlResult etlResult = new EtlResult(); + AtomicLong impCount = new AtomicLong(); + List errMsg = new ArrayList<>(); + String esIndex = ""; + if (config == null) { + logger.warn("esSycnCofnig is null, etl go end !"); + etlResult.setErrorMessage("esSycnCofnig is null, etl go end !"); + return etlResult; + } + + ESMapping mapping = config.getEsMapping(); + + esIndex = mapping.get_index(); + DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$"); + Matcher matcher = pattern.matcher(dataSource.getUrl()); + if (!matcher.find()) { + throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey()); + } + String schema = matcher.group(2); + + logger.info("etl from db: {}, to es index: {}", schema, esIndex); + long start = System.currentTimeMillis(); + try { + String sql = mapping.getSql(); + + // 拼接条件 + if (mapping.getEtlCondition() != null && params != null) { + String etlCondition = mapping.getEtlCondition(); + int size = params.size(); + for (int i = 0; i < size; i++) { + etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); + } + + sql += " " + etlCondition; + } + + if (logger.isDebugEnabled()) { + logger.debug("etl sql : {}", mapping.getSql()); + } + + if (bulk) { + // 获取总数 + String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; + long cnt = (Long) ESSyncUtil.sqlRS(dataSource, countSql, rs -> { + Long count = null; + try { + if (rs.next()) { + count = ((Number) rs.getObject(1)).longValue(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return count == null ? 0L : count; + }); + + // 当大于1万条记录时开启多线程 + if (cnt >= 10000) { + int threadCount = 3; // TODO 从配置读取默认为3 + long perThreadCnt = cnt / threadCount; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> futures = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; i++) { + long offset = i * perThreadCnt; + Long size = null; + if (i != threadCount - 1) { + size = perThreadCnt; + } + String sqlFinal; + if (size != null) { + sqlFinal = sql + " LIMIT " + offset + "," + size; + } else { + sqlFinal = sql + " LIMIT " + offset + "," + cnt; + } + Future future = executor + .submit(() -> executeSqlImport(dataSource, sqlFinal, mapping, impCount, errMsg)); + futures.add(future); + } + + for (Future future : futures) { + future.get(); + } + + executor.shutdown(); + } else { + executeSqlImport(dataSource, sql, mapping, impCount, errMsg); + } + } else { + logger.info("自动ETL，无需统计记录总条数，直接进行ETL, index: {}", esIndex); + executeSqlImport(dataSource, sql, mapping, impCount, errMsg); + } + + logger.info("数据全量导入完成,一共导入 {} 条数据, 耗时: {}", impCount.get(), System.currentTimeMillis() - start); + etlResult.setResultMessage("导入ES索引 " + esIndex + " 数据：" + impCount.get() + " 条"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(esIndex + " etl failed! ==>" + e.getMessage()); + } + if (errMsg.isEmpty()) { + etlResult.setSucceeded(true); + } else { + etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + } + return etlResult; + } + + private void processFailBulkResponse(BulkResponse bulkResponse, boolean hasParent) { + for (BulkItemResponse response : bulkResponse.getItems()) { + if (!response.isFailed()) { + continue; + } + + if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.warn(response.getFailureMessage()); + } else { + logger.error("全量导入数据有误 {}", response.getFailureMessage()); + throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage()); + } + } + } + + private boolean executeSqlImport(DataSource ds, String sql, ESMapping mapping, AtomicLong impCount, + List errMsg) { + try { + ESSyncUtil.sqlRS(ds, sql, rs -> { + int count = 0; + try { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + + long batchBegin = System.currentTimeMillis(); + while (rs.next()) { + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) { + + // 如果是主键字段则不插入 + if (fieldItem.getFieldName().equals(mapping.get_id())) { + continue; + } + + String fieldName = fieldItem.getFieldName(); + if (mapping.getSkips().contains(fieldName)) { + continue; + } + + Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName); + esFieldData.put(fieldName, val); + } + Object idVal = null; + if (mapping.get_id() != null) { + idVal = rs.getObject(mapping.get_id()); + } + + if (idVal != null) { + if (mapping.getParent() == null) { + bulkRequestBuilder.add(transportClient + .prepareIndex(mapping.get_index(), mapping.get_type(), idVal.toString()) + .setSource(esFieldData)); + } else { + // ignore + } + } else { + if (mapping.getParent() == null) { + // TODO 删除pk对应的数据 + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()) + .setSource(esFieldData)); + } else { + // ignore + } + } + + if (bulkRequestBuilder.numberOfActions() % mapping.getCommitBatch() == 0 + && bulkRequestBuilder.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + BulkResponse rp = bulkRequestBuilder.execute().actionGet(); + if (rp.hasFailures()) { + this.processFailBulkResponse(rp, Objects.nonNull(mapping.getParent())); + } + + if (logger.isDebugEnabled()) { + logger.debug("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + bulkRequestBuilder.numberOfActions(), + mapping.get_index()); + } + batchBegin = System.currentTimeMillis(); + bulkRequestBuilder = transportClient.prepareBulk(); + } + count++; + impCount.incrementAndGet(); + } + + if (bulkRequestBuilder.numberOfActions() > 0) { + long esBatchBegin = System.currentTimeMillis(); + BulkResponse rp = bulkRequestBuilder.execute().actionGet(); + if (rp.hasFailures()) { + this.processFailBulkResponse(rp, Objects.nonNull(mapping.getParent())); + } + if (logger.isDebugEnabled()) { + logger.debug("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", + (System.currentTimeMillis() - batchBegin), + (System.currentTimeMillis() - esBatchBegin), + bulkRequestBuilder.numberOfActions(), + mapping.get_index()); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage()); + throw new RuntimeException(e); + } + return count; + }); + + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index cbab4c64..a77aaeef 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -75,6 +75,11 @@ public class ESSyncService { public void sync(ESSyncConfig config, Dml dml) { try { + // 如果是按时间戳定时更新则返回 + if (config.getEsMapping().isSyncByTimestamp()) { + return; + } + long begin = System.currentTimeMillis(); String type = dml.getType(); @@ -98,6 +103,213 @@ public class ESSyncService { } } + /** + * 插入操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void insert(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + if (dataList == null || dataList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + for (Map data : dataList) { + if (data == null || data.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledInsert(config, dml, data); + } else { + // ------是主表 查询sql来插入------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + mainTableInsert(config, dml, data); + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段插入------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段插入------ + subTableSimpleFieldOperation(config, dml, data, null, tableItem); + } + } else { + // ------关联子表复杂字段插入 执行全sql更新es------ + wholeSqlOperation(config, dml, data, null, tableItem); + } + } + } + } + } + + /** + * 更新操作dml + * + * @param config es配置 + * @param dml dml数据 + */ + private void update(ESSyncConfig config, Dml dml) { + List> dataList = dml.getData(); + List> oldList = dml.getOld(); + if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) { + return; + } + SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); + int i = 0; + for (Map data : dataList) { + Map old = oldList.get(i); + if (data == null || data.isEmpty() || old == null || old.isEmpty()) { + continue; + } + + if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { + // ------单表 & 所有字段都为简单字段------ + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + // ------主表 查询sql来更新------ + if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { + ESMapping mapping = config.getEsMapping(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName); + + boolean idFieldSimple = true; + if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) { + idFieldSimple = false; + } + + boolean allUpdateFieldSimple = true; + out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (old.containsKey(columnItem.getColumnName())) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allUpdateFieldSimple = false; + break out; + } + } + } + } + + // 不支持主键更新!! + + // 判断是否有外键更新 + boolean fkChanged = false; + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + boolean changed = false; + for (List fieldItems : tableItem.getRelationTableFields().values()) { + for (FieldItem fieldItem : fieldItems) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + fkChanged = true; + changed = true; + break; + } + } + } + // 如果外键有修改,则更新所对应该表的所有查询条件数据 + if (changed) { + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + fieldItem.getColumnItems() + .forEach(columnItem -> old.put(columnItem.getColumnName(), null)); + } + } + } + + // 判断主键和所更新的字段是否全为简单字段 + if (idFieldSimple && allUpdateFieldSimple && !fkChanged) { + singleTableSimpleFiledUpdate(config, dml, data, old); + } else { + mainTableUpdate(config, dml, data, old); + } + } + + // 从表的操作 + for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { + if (tableItem.isMain()) { + continue; + } + if (!tableItem.getTableName().equals(dml.getTable())) { + continue; + } + + // 关联条件出现在主表查询条件是否为简单字段 + boolean allFieldsSimple = true; + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + + // 所有查询字段均为简单字段 + if (allFieldsSimple) { + // 不是子查询 + if (!tableItem.isSubQuery()) { + // ------关联表简单字段更新------ + Map esFieldData = new LinkedHashMap<>(); + for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { + if (old.containsKey(fieldItem.getColumn().getColumnName())) { + Object value = esTemplate.getValFromData(config.getEsMapping(), + data, + fieldItem.getFieldName(), + fieldItem.getColumn().getColumnName()); + esFieldData.put(fieldItem.getFieldName(), value); + } + } + joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); + } else { + // ------关联子表简单字段更新------ + subTableSimpleFieldOperation(config, dml, data, old, tableItem); + } + } else { + // ------关联子表复杂字段更新 执行全sql更新es------ + wholeSqlOperation(config, dml, data, old, tableItem); + } + } + } + + i++; + } + } + + /** + * 删除操作dml + * + * @param config es配置 + * @param dml dml数据 + */ private void delete(ESSyncConfig config, Dml dml) { List> dataList = dml.getData(); if (dataList == null || dataList.isEmpty()) { @@ -177,184 +389,12 @@ public class ESSyncService { } } else { // ------关联子表复杂字段更新 执行全sql更新es------ - jonTableWholeSqlOperation(config, dml, data, null, tableItem); + wholeSqlOperation(config, dml, data, null, tableItem); } } } } - /** - * 插入操作dml - * - * @param config es配置 - * @param dml dml数据 - */ - private void insert(ESSyncConfig config, Dml dml) { - List> dataList = dml.getData(); - if (dataList == null || dataList.isEmpty()) { - return; - } - SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); - for (Map data : dataList) { - if (data == null || data.isEmpty()) { - continue; - } - - if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { - // ------单表 & 所有字段都为简单字段------ - singleTableSimpleFiledInsert(config, dml, data); - } else { - // ------是主表 查询sql来插入------ - if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { - mainTableInsert(config, dml, data); - } - - // 从表的操作 - for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { - if (tableItem.isMain()) { - continue; - } - if (!tableItem.getTableName().equals(dml.getTable())) { - continue; - } - // 关联条件出现在主表查询条件是否为简单字段 - boolean allFieldsSimple = true; - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { - allFieldsSimple = false; - break; - } - } - // 所有查询字段均为简单字段 - if (allFieldsSimple) { - // 不是子查询 - if (!tableItem.isSubQuery()) { - // ------关联表简单字段插入------ - Map esFieldData = new LinkedHashMap<>(); - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - Object value = esTemplate.getValFromData(config.getEsMapping(), - data, - fieldItem.getFieldName(), - fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), value); - } - - joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); - } else { - // ------关联子表简单字段插入------ - subTableSimpleFieldOperation(config, dml, data, null, tableItem); - } - } else { - // ------关联子表复杂字段插入 执行全sql更新es------ - jonTableWholeSqlOperation(config, dml, data, null, tableItem); - } - } - } - } - } - - /** - * 更新操作dml - * - * @param config es配置 - * @param dml dml数据 - */ - private void update(ESSyncConfig config, Dml dml) { - List> dataList = dml.getData(); - List> oldList = dml.getOld(); - if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) { - return; - } - SchemaItem schemaItem = config.getEsMapping().getSchemaItem(); - int i = 0; - for (Map data : dataList) { - Map old = oldList.get(i); - if (data == null || data.isEmpty() || old == null || old.isEmpty()) { - continue; - } - - if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) { - // ------单表 & 所有字段都为简单字段------ - singleTableSimpleFiledUpdate(config, dml, data, old); - } else { - // ------主表 查询sql来更新------ - if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) { - ESMapping mapping = config.getEsMapping(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName); - boolean idFieldSimple = true; - if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) { - idFieldSimple = false; - } - - boolean allUpdateFieldSimple = true; - out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - for (ColumnItem columnItem : fieldItem.getColumnItems()) { - if (old.containsKey(columnItem.getColumnName())) { - if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { - allUpdateFieldSimple = false; - break out; - } - } - } - } - // 判断主键和所更新的字段是否全为简单字段 - if (idFieldSimple && allUpdateFieldSimple) { - singleTableSimpleFiledUpdate(config, dml, data, old); - } else { - mainTableUpdate(config, dml, data, old); - } - } - - // 从表的操作 - for (TableItem tableItem : schemaItem.getAliasTableItems().values()) { - if (tableItem.isMain()) { - continue; - } - if (!tableItem.getTableName().equals(dml.getTable())) { - continue; - } - - // 关联条件出现在主表查询条件是否为简单字段 - boolean allFieldsSimple = true; - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { - allFieldsSimple = false; - break; - } - } - - // 所有查询字段均为简单字段 - if (allFieldsSimple) { - // 不是子查询 - if (!tableItem.isSubQuery()) { - // ------关联表简单字段更新------ - Map esFieldData = new LinkedHashMap<>(); - for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { - if (old.containsKey(fieldItem.getColumn().getColumnName())) { - Object value = esTemplate.getValFromData(config.getEsMapping(), - data, - fieldItem.getFieldName(), - fieldItem.getColumn().getColumnName()); - esFieldData.put(fieldItem.getFieldName(), value); - } - } - joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData); - } else { - // ------关联子表简单字段更新------ - subTableSimpleFieldOperation(config, dml, data, old, tableItem); - } - } else { - // ------关联子表复杂字段更新 执行全sql更新es------ - jonTableWholeSqlOperation(config, dml, data, old, tableItem); - } - } - } - - i++; - } - } - /** * 单表简单字段insert * @@ -633,8 +673,8 @@ public class ESSyncService { * @param data 单行dml数据 * @param tableItem 当前表配置 */ - private void jonTableWholeSqlOperation(ESSyncConfig config, Dml dml, Map data, - Map old, TableItem tableItem) { + private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map data, Map old, + TableItem tableItem) { ESMapping mapping = config.getEsMapping(); StringBuilder sql = new StringBuilder(mapping.getSql() + " WHERE "); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 655087e7..4271e937 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -330,7 +330,7 @@ public class ESTemplate { } for (ColumnItem columnItem : fieldItem.getColumnItems()) { - if (dmlOld.get(columnItem.getColumnName()) != null + if (dmlOld.containsKey(columnItem.getColumnName()) && !mapping.getSkips().contains(fieldItem.getFieldName())) { esFieldData.put(fieldItem.getFieldName(), getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 81dd7b3d..9fa43bd3 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -10,5 +10,7 @@ esMapping: left join role b on b.id=a.role_id left join (select user_id, group_concat(label order by id desc separator ';') as labels from label group by user_id) c on c.user_id=a.id" +# syncByTimestamp: false +# syncInterval: 2000 # ms + etlCondition: "where a.c_time>='{0}'" commitBatch: 3000 - etlCondition: "where a.c_time>='{0}'" \ No newline at end of file From 347601b9fe9d6f0d185d6c9f8e8a07ee95c68f3c Mon Sep 17 00:00:00 2001 From: mcy Date: Mon, 5 Nov 2018 11:40:27 +0800 Subject: [PATCH 35/42] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/adapter/es/test/sync/Common.java | 47 +++++++++- .../es/test/sync/LabelSyncJoinSub2Test.java | 41 ++++++--- .../es/test/sync/LabelSyncJoinSubTest.java | 39 +++++--- .../es/test/sync/RoleSyncJoinOne2Test.java | 28 +++--- .../es/test/sync/RoleSyncJoinOneTest.java | 92 ++++++++++++++++--- .../es/test/sync/UserSyncJoinOneTest.java | 30 +++--- .../es/test/sync/UserSyncSingleTest.java | 16 ++-- .../client/adapter/es/test/sync/db_schema.sql | 39 ++++++++ .../adapter/es/test/sync/es_mapping.json | 21 +++++ 9 files changed, 280 insertions(+), 73 deletions(-) create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql create mode 100644 client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java index e5b31cbd..10d41de1 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java @@ -1,13 +1,18 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import javax.sql.DataSource; + import com.alibaba.otter.canal.client.adapter.es.ESAdapter; import com.alibaba.otter.canal.client.adapter.es.test.TestConstant; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import java.util.HashMap; -import java.util.Map; - public class Common { public static ESAdapter init() { @@ -24,4 +29,40 @@ public class Common { esAdapter.init(outerAdapterConfig); return esAdapter; } + + public static void sqlExe(DataSource dataSource, String sql) { + Connection conn = null; + Statement stmt = null; + try { + conn = dataSource.getConnection(); + conn.setAutoCommit(false); + stmt = conn.createStatement(); + stmt.execute(sql); + conn.commit(); + } catch (Exception e) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e1) { + // ignore + } + } + e.printStackTrace(); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // ignore + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // ignore + } + } + } + } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java index f418c1bf..cb947de9 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java @@ -14,6 +14,8 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; +import javax.sql.DataSource; + public class LabelSyncJoinSub2Test { private ESAdapter esAdapter; @@ -25,10 +27,15 @@ public class LabelSyncJoinSub2Test { } /** - * 子查询从表插入 (确保主表记录必须有数据) + * 带函数子查询从表插入 */ @Test - public void insertTest01() { + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"delete from label where id=1 or id=2"); + Common.sqlExe(ds,"insert into label (id,user_id,label) values (1,1,'a')"); + Common.sqlExe(ds,"insert into label (id,user_id,label) values (2,1,'b')"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -38,20 +45,26 @@ public class LabelSyncJoinSub2Test { List> dataList = new ArrayList<>(); Map data = new LinkedHashMap<>(); dataList.add(data); - data.put("id", 1L); + data.put("id", 2L); data.put("user_id",1L); - data.put("label", "a"); + data.put("label", "b"); dml.setData(dataList); esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("a;b_", response.getSource().get("_labels")); + Assert.assertEquals("b;a_", response.getSource().get("_labels")); } + /** + * 带函数子查询从表更新 + */ @Test - public void updateTest02() { + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"update label set label='aa' where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -75,11 +88,17 @@ public class LabelSyncJoinSub2Test { esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("aa;b_", response.getSource().get("_labels")); + Assert.assertEquals("b;aa_", response.getSource().get("_labels")); } + /** + * 带函数子查询从表删除 + */ @Test - public void deleteTest03() { + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"delete from label where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -100,10 +119,4 @@ public class LabelSyncJoinSub2Test { GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); Assert.assertEquals("b_", response.getSource().get("_labels")); } - - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); - } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java index 9c7944b0..bcf4d372 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java @@ -14,6 +14,8 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; +import javax.sql.DataSource; + public class LabelSyncJoinSubTest { private ESAdapter esAdapter; @@ -25,10 +27,15 @@ public class LabelSyncJoinSubTest { } /** - * 子查询从表插入 (确保主表记录必须有数据) + * 子查询从表插入 */ @Test - public void insertTest01() { + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"delete from label where id=1 or id=2"); + Common.sqlExe(ds,"insert into label (id,user_id,label) values (1,1,'a')"); + Common.sqlExe(ds,"insert into label (id,user_id,label) values (2,1,'b')"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -38,9 +45,9 @@ public class LabelSyncJoinSubTest { List> dataList = new ArrayList<>(); Map data = new LinkedHashMap<>(); dataList.add(data); - data.put("id", 1L); + data.put("id", 2L); data.put("user_id",1L); - data.put("label", "a"); + data.put("label", "b"); dml.setData(dataList); @@ -50,8 +57,14 @@ public class LabelSyncJoinSubTest { Assert.assertEquals("b;a", response.getSource().get("_labels")); } + /** + * 子查询从表更新 + */ @Test - public void updateTest02() { + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"update label set label='aa' where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -75,11 +88,17 @@ public class LabelSyncJoinSubTest { esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("aa;b", response.getSource().get("_labels")); + Assert.assertEquals("b;aa", response.getSource().get("_labels")); } + /** + * 子查询从表删除 + */ @Test - public void deleteTest03() { + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"delete from label where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -100,10 +119,4 @@ public class LabelSyncJoinSubTest { GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); Assert.assertEquals("b", response.getSource().get("_labels")); } - - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); - } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java index 42e119db..a8fdf119 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java @@ -14,6 +14,8 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; +import javax.sql.DataSource; + public class RoleSyncJoinOne2Test { private ESAdapter esAdapter; @@ -25,10 +27,14 @@ public class RoleSyncJoinOne2Test { } /** - * 非子查询从表插入 (确保主表记录必须有数据) + * 带函数非子查询从表插入 */ @Test - public void insertTest01() { + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"delete from role where id=1"); + Common.sqlExe(ds,"insert into role (id,role_name) values (1,'admin')"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -39,18 +45,24 @@ public class RoleSyncJoinOne2Test { Map data = new LinkedHashMap<>(); dataList.add(data); data.put("id", 1L); - data.put("role_name", "admin2"); + data.put("role_name", "admin"); dml.setData(dataList); esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("admin2_", response.getSource().get("_role_name")); + Assert.assertEquals("admin_", response.getSource().get("_role_name")); } + /** + * 带函数非子查询从表更新 + */ @Test - public void updateTest02() { + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"update role set role_name='admin3' where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -75,10 +87,4 @@ public class RoleSyncJoinOne2Test { GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); Assert.assertEquals("admin3_", response.getSource().get("_role_name")); } - - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); - } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java index aa383684..9c5ab300 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java @@ -2,13 +2,13 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync; import java.util.*; +import javax.sql.DataSource; + import org.elasticsearch.action.get.GetResponse; -import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.otter.canal.client.adapter.es.ESAdapter; import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; @@ -25,10 +25,14 @@ public class RoleSyncJoinOneTest { } /** - * 非子查询从表插入 (确保主表记录必须有数据) + * 非子查询从表插入 */ @Test - public void insertTest01() { + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -49,8 +53,14 @@ public class RoleSyncJoinOneTest { Assert.assertEquals("admin", response.getSource().get("_role_name")); } + /** + * 非子查询从表更新 + */ @Test - public void updateTest02() { + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "update role set role_name='admin2' where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -76,8 +86,72 @@ public class RoleSyncJoinOneTest { Assert.assertEquals("admin2", response.getSource().get("_role_name")); } + /** + * 主表更新外键值 + */ @Test - public void deleteTest03() { + public void test03() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=2"); + Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')"); + Common.sqlExe(ds, "update user set role_id=2 where id=1"); + + Dml dml = new Dml(); + dml.setDestination("example"); + dml.setTs(new Date().getTime()); + dml.setType("UPDATE"); + dml.setDatabase("mytest"); + dml.setTable("user"); + List> dataList = new ArrayList<>(); + Map data = new LinkedHashMap<>(); + dataList.add(data); + data.put("id", 1L); + data.put("role_id", 2L); + dml.setData(dataList); + List> oldList = new ArrayList<>(); + Map old = new LinkedHashMap<>(); + oldList.add(old); + old.put("role_id", 1L); + dml.setOld(oldList); + esAdapter.getEsSyncService().sync(dml); + + GetResponse response = + esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("operator", response.getSource().get("_role_name")); + + Common.sqlExe(ds, "update user set role_id=1 where id=1"); + + Dml dml2 = new Dml(); + dml2.setDestination("example"); + dml2.setTs(new Date().getTime()); + dml2.setType("UPDATE"); + dml2.setDatabase("mytest"); + dml2.setTable("user"); + List> dataList2 = new ArrayList<>(); + Map data2 = new LinkedHashMap<>(); + dataList2.add(data2); + data2.put("id", 1L); + data2.put("role_id", 1L); + dml2.setData(dataList2); + List> oldList2 = new ArrayList<>(); + Map old2 = new LinkedHashMap<>(); + oldList2.add(old2); + old2.put("role_id", 2L); + dml2.setOld(oldList2); + esAdapter.getEsSyncService().sync(dml2); + + GetResponse response2 = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); + Assert.assertEquals("admin2", response2.getSource().get("_role_name")); + } + + /** + * 非子查询从表删除 + */ + @Test + public void test04() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds, "delete from role where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -97,10 +171,4 @@ public class RoleSyncJoinOneTest { GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); Assert.assertNull(response.getSource().get("_role_name")); } - - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); - } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java index 2406f301..d34d61b2 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java @@ -2,6 +2,8 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync; import java.util.*; +import javax.sql.DataSource; + import org.elasticsearch.action.get.GetResponse; import org.junit.After; import org.junit.Assert; @@ -25,10 +27,14 @@ public class UserSyncJoinOneTest { } /** - * 主表带函数插入, 数据库里内容必须和单测一致 + * 主表带函数插入 */ @Test - public void insertTest01() { + public void test01() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"delete from user where id=1"); + Common.sqlExe(ds,"insert into user (id,name,role_id) values (1,'Eric',1)"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -51,8 +57,14 @@ public class UserSyncJoinOneTest { Assert.assertEquals("Eric_", response.getSource().get("_name")); } + /** + * 主表带函数更新 + */ @Test - public void updateTest02() { + public void test02() { + DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS"); + Common.sqlExe(ds,"update user set name='Eric2' where id=1"); + Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -63,23 +75,17 @@ public class UserSyncJoinOneTest { Map data = new LinkedHashMap<>(); dataList.add(data); data.put("id", 1L); - data.put("name", "Eric"); + data.put("name", "Eric2"); dml.setData(dataList); List> oldList = new ArrayList<>(); Map old = new LinkedHashMap<>(); oldList.add(old); - old.put("name", "Eric2"); + old.put("name", "Eric"); dml.setOld(oldList); esAdapter.getEsSyncService().sync(dml); GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get(); - Assert.assertEquals("Eric_", response.getSource().get("_name")); - } - - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + Assert.assertEquals("Eric2_", response.getSource().get("_name")); } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java index 98fa1d6b..0b98a2b0 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java @@ -28,7 +28,7 @@ public class UserSyncSingleTest { * 单表插入 */ @Test - public void insertTest01() { + public void test01() { Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -55,7 +55,7 @@ public class UserSyncSingleTest { * 单表更新 */ @Test - public void updateTest02() { + public void test02() { Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -84,7 +84,7 @@ public class UserSyncSingleTest { * 单表删除 */ @Test - public void deleteTest03() { + public void test03() { Dml dml = new Dml(); dml.setDestination("example"); dml.setTs(new Date().getTime()); @@ -107,9 +107,9 @@ public class UserSyncSingleTest { Assert.assertNull(response.getSource()); } - @After - public void after() { - esAdapter.destroy(); - DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); - } + // @After + // public void after() { + // esAdapter.destroy(); + // DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close); + // } } diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql new file mode 100644 index 00000000..bff2e118 --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql @@ -0,0 +1,39 @@ +-- ---------------------------- +-- Table structure for label +-- ---------------------------- +DROP TABLE IF EXISTS `label`; +CREATE TABLE `label` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `user_id` bigint(20) NOT NULL, + `label` varchar(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for role +-- ---------------------------- +DROP TABLE IF EXISTS `role`; +CREATE TABLE `role` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `role_name` varchar(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for user +-- ---------------------------- +DROP TABLE IF EXISTS `user`; +CREATE TABLE `user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(30) NOT NULL, + `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `role_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +insert into user (id,name,role_id) values (1,'Eric',1); +insert into role (id,role_name) values (1,'admin'); +insert into role (id,role_name) values (2,'operator'); +insert into label (id,user_id,label) values (1,1,'a'); +insert into label (id,user_id,label) values (2,1,'b'); +commit; \ No newline at end of file diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json new file mode 100644 index 00000000..55b0389a --- /dev/null +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json @@ -0,0 +1,21 @@ +{ + "_doc": { + "properties": { + "_name": { + "type": "text" + }, + "_role_id": { + "type": "long" + }, + "_role_name": { + "type": "text" + }, + "_labels": { + "type": "text" + }, + "_c_time": { + "type": "date" + } + } + } +} \ No newline at end of file From 650446de5b2324ae7ba10049f2c50fe605e374a7 Mon Sep 17 00:00:00 2001 From: mcy Date: Mon, 5 Nov 2018 13:36:35 +0800 Subject: [PATCH 36/42] =?UTF-8?q?etl=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 72 +++++++++++++++++-- .../adapter/es/service/ESEtlService.java | 15 +++- .../client/adapter/es/support/ESTemplate.java | 2 +- .../src/main/resources/es/mytest_user.yml | 2 - 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index e5cc318b..3f11035a 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -1,9 +1,13 @@ package com.alibaba.otter.canal.client.adapter.es; import java.net.InetAddress; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import javax.sql.DataSource; + +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -12,13 +16,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; +import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.SPI; +import com.alibaba.otter.canal.client.adapter.support.*; /** * ES外部适配器 @@ -73,12 +77,64 @@ public class ESAdapter implements OuterAdapter { @Override public EtlResult etl(String task, List params) { - return null; + EtlResult etlResult = new EtlResult(); + ESSyncConfig config = ESSyncConfigLoader.getEsSyncConfig().get(task); + if (config != null) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + ESEtlService esEtlService = new ESEtlService(transportClient, config); + if (dataSource != null) { + return esEtlService.importData(params, false); + } else { + etlResult.setSucceeded(false); + etlResult.setErrorMessage("DataSource not found"); + return etlResult; + } + } else { + StringBuilder resultMsg = new StringBuilder(); + boolean resSuccess = true; + // ds不为空说明传入的是datasourceKey + for (ESSyncConfig configTmp : ESSyncConfigLoader.getEsSyncConfig().values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + ESEtlService esEtlService = new ESEtlService(transportClient, configTmp); + EtlResult etlRes = esEtlService.importData(params, false); + if (!etlRes.getSucceeded()) { + resSuccess = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); + } else { + resultMsg.append(etlRes.getResultMessage()).append("\n"); + } + } + } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSuccess); + if (resSuccess) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } + } + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Task not found"); + return etlResult; } @Override public Map count(String task) { - return null; + ESSyncConfig config = ESSyncConfigLoader.getEsSyncConfig().get(task); + ESMapping mapping = config.getEsMapping(); + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setSize(0) + .get(); + + long rowCount = response.getHits().getTotalHits(); + Map res = new LinkedHashMap<>(); + res.put("esIndex", mapping.get_index()); + res.put("count", rowCount); + return res; } @Override @@ -90,6 +146,10 @@ public class ESAdapter implements OuterAdapter { @Override public String getDestination(String task) { + ESSyncConfig config = ESSyncConfigLoader.getEsSyncConfig().get(task); + if (config != null) { + return config.getDestination(); + } return null; } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java index 12a41a73..d7045e47 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java @@ -13,8 +13,11 @@ import javax.sql.DataSource; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -204,8 +207,18 @@ public class ESEtlService { // ignore } } else { + idVal = rs.getObject(mapping.getPk()); if (mapping.getParent() == null) { - // TODO 删除pk对应的数据 + // 删除pk对应的数据 + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder.add(transportClient + .prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + bulkRequestBuilder .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()) .setSource(esFieldData)); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 4271e937..9ec95d54 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -40,7 +40,7 @@ public class ESTemplate { private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); - private static final int MAX_BATCH_SIZE = 1000; + private static final int MAX_BATCH_SIZE = 10000; private TransportClient transportClient; diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 9fa43bd3..7b9effde 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -10,7 +10,5 @@ esMapping: left join role b on b.id=a.role_id left join (select user_id, group_concat(label order by id desc separator ';') as labels from label group by user_id) c on c.user_id=a.id" -# syncByTimestamp: false -# syncInterval: 2000 # ms etlCondition: "where a.c_time>='{0}'" commitBatch: 3000 From 6ff57f53f8169caa1202a1d63caca0c699a6ae2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Mon, 5 Nov 2018 15:06:40 +0800 Subject: [PATCH 37/42] fixed issue #1070, support mysql8.0.13 --- .../tddl/dbsync/binlog/JsonConversion.java | 82 +++-- .../dbsync/binlog/JsonDiffConversion.java | 154 ++++++++ .../taobao/tddl/dbsync/binlog/LogBuffer.java | 4 +- .../taobao/tddl/dbsync/binlog/LogDecoder.java | 8 + .../taobao/tddl/dbsync/binlog/LogEvent.java | 10 +- .../event/FormatDescriptionLogEvent.java | 1 + .../dbsync/binlog/event/QueryLogEvent.java | 101 ++++- .../dbsync/binlog/event/RowsLogBuffer.java | 86 +++-- .../dbsync/binlog/event/RowsLogEvent.java | 25 +- .../dbsync/binlog/event/TableMapLogEvent.java | 348 +++++++++++++++++- .../binlog/event/UpdateRowsLogEvent.java | 7 +- .../dbsync/binlog/DirectLogFetcherTest.java | 1 + .../dbsync/binlog/FileLogFetcherTest.java | 1 + .../parse/driver/mysql/utils/CharsetUtil.java | 258 +++++++++++++ .../parse/driver/mysql/CharsetUtilTest.java | 51 +++ .../mysql/MysqlMultiStageCoprocessor.java | 1 + .../inbound/mysql/dbsync/LogEventConvert.java | 60 ++- .../canal/parse/DirectLogFetcherTest.java | 124 ++++++- .../MysqlBinlogParsePerformanceTest.java | 5 +- 19 files changed, 1215 insertions(+), 112 deletions(-) create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java create mode 100644 driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java index 020591f7..e61b6645 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java @@ -1,5 +1,9 @@ package com.taobao.tddl.dbsync.binlog; +import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2; +import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4; +import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr; + /** * 处理下MySQL json二进制转化为可读的字符串 * @@ -330,12 +334,30 @@ public class JsonConversion { long ultime = Math.abs(packed_value); long intpart = ultime >> 24; int frac = (int) (ultime % (1L << 24)); - text = String.format("%s%02d:%02d:%02d", - packed_value >= 0 ? "" : "-", - (int) ((intpart >> 12) % (1 << 10)), - (int) ((intpart >> 6) % (1 << 6)), - (int) (intpart % (1 << 6))); - text = text + "." + usecondsToStr(frac, 6); + // text = String.format("%s%02d:%02d:%02d", + // packed_value >= 0 ? "" : "-", + // (int) ((intpart >> 12) % (1 << 10)), + // (int) ((intpart >> 6) % (1 << 6)), + // (int) (intpart % (1 << 6))); + // text = text + "." + usecondsToStr(frac, 6); + StringBuilder builder = new StringBuilder(17); + if (packed_value < 0) { + builder.append('-'); + } + + int d = (int) ((intpart >> 12) % (1 << 10)); + if (d > 100) { + builder.append(String.valueOf(d)); + } else { + appendNumber2(builder, d); + } + builder.append(':'); + appendNumber2(builder, (int) ((intpart >> 6) % (1 << 6))); + builder.append(':'); + appendNumber2(builder, (int) (intpart % (1 << 6))); + + builder.append('.').append(usecondsToStr(frac, 6)); + text = builder.toString(); } buf.append('"').append(text).append('"'); } else if (m_field_type == LogEvent.MYSQL_TYPE_DATE || m_field_type == LogEvent.MYSQL_TYPE_DATETIME @@ -351,14 +373,28 @@ public class JsonConversion { long ymd = intpart >> 17; long ym = ymd >> 5; long hms = intpart % (1 << 17); - text = String.format("%04d-%02d-%02d %02d:%02d:%02d", - (int) (ym / 13), - (int) (ym % 13), - (int) (ymd % (1 << 5)), - (int) (hms >> 12), - (int) ((hms >> 6) % (1 << 6)), - (int) (hms % (1 << 6))); - text = text + "." + usecondsToStr(frac, 6); + // text = + // String.format("%04d-%02d-%02d %02d:%02d:%02d", + // (int) (ym / 13), + // (int) (ym % 13), + // (int) (ymd % (1 << 5)), + // (int) (hms >> 12), + // (int) ((hms >> 6) % (1 << 6)), + // (int) (hms % (1 << 6))); + StringBuilder builder = new StringBuilder(26); + appendNumber4(builder, (int) (ym / 13)); + builder.append('-'); + appendNumber2(builder, (int) (ym % 13)); + builder.append('-'); + appendNumber2(builder, (int) (ymd % (1 << 5))); + builder.append(' '); + appendNumber2(builder, (int) (hms >> 12)); + builder.append(':'); + appendNumber2(builder, (int) ((hms >> 6) % (1 << 6))); + builder.append(':'); + appendNumber2(builder, (int) (hms % (1 << 6))); + builder.append('.').append(usecondsToStr(frac, 6)); + text = builder.toString(); } buf.append('"').append(text).append('"'); } else { @@ -397,22 +433,4 @@ public class JsonConversion { OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR } - private static String usecondsToStr(int frac, int meta) { - String sec = String.valueOf(frac); - if (meta > 6) { - throw new IllegalArgumentException("unknow useconds meta : " + meta); - } - - if (sec.length() < 6) { - StringBuilder result = new StringBuilder(6); - int len = 6 - sec.length(); - for (; len > 0; len--) { - result.append('0'); - } - result.append(sec); - sec = result.toString(); - } - - return sec.substring(0, meta); - } } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java new file mode 100644 index 00000000..223b5d9c --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java @@ -0,0 +1,154 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.util.ArrayList; +import java.util.List; + +import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value; +import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_enum_type; + +/** + * 处理mysql8.0 parital json diff解析 + * + * @author agapple 2018年11月4日 下午3:53:46 + * @since 1.1.2 + */ +public class JsonDiffConversion { + + /** + * The JSON value in the given path is replaced with a new value. It has the + * same effect as `JSON_REPLACE(col, path, value)`. + */ + public static final int DIFF_OPERATION_REPLACE = 0; + /** + * Add a new element at the given path. If the path specifies an array + * element, it has the same effect as `JSON_ARRAY_INSERT(col, path, value)`. + * If the path specifies an object member, it has the same effect as + * `JSON_INSERT(col, path, value)`. + */ + public static final int DIFF_OPERATION_INSERT = 1; + /** + * The JSON value at the given path is removed from an array or object. It + * has the same effect as `JSON_REMOVE(col, path)`. + */ + public static final int DIFF_OPERATION_REMOVE = 2; + + public static final int JSON_DIFF_OPERATION_COUNT = 3; + + public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex, + String charsetName) { + int position = buffer.position(); + List operation_names = new ArrayList(); + while (buffer.hasRemaining()) { + int operation_int = buffer.getUint8(); + if (operation_int >= JSON_DIFF_OPERATION_COUNT) { + throw new IllegalArgumentException("reading operation type (invalid operation code)"); + } + + // skip path + long path_length = buffer.getPackedLong(); + if (path_length > len) { + throw new IllegalArgumentException("skipping path"); + } + + // compute operation name + byte[] lastP = buffer.getData(buffer.position() + (int) path_length - 1, 1); + String operation_name = json_diff_operation_name(operation_int, lastP[0]); + operation_names.add(operation_name); + + buffer.forward((int) path_length); + // skip value + if (operation_int != DIFF_OPERATION_REMOVE) { + long value_length = buffer.getPackedLong(); + if (value_length > len) { + throw new IllegalArgumentException("skipping path"); + } + + buffer.forward((int) value_length); + } + } + + // Print function names in reverse order. + StringBuilder builder = new StringBuilder(); + for (int i = operation_names.size() - 1; i >= 0; i--) { + if (i == 0 || operation_names.get(i - 1) != operation_names.get(i)) { + builder.append(operation_names.get(i)).append("("); + } + } + + // Print column id + if (columnName != null) { + builder.append(columnName); + } else { + builder.append("@").append(columnIndex); + } + + // In case this vector is empty (a no-op), make an early return + // after printing only the column name + if (operation_names.size() == 0) { + return builder; + } + + // Print comma between column name and next function argument + builder.append(", "); + // Print paths and values. + buffer.position(position); + int diff_i = 0; + while (buffer.hasRemaining()) { + // Read operation + int operation_int = buffer.getUint8(); + + // Read path length + long path_length = buffer.getPackedLong(); + // Print path + builder.append('\'').append(buffer.getFixString((int) path_length)).append('\''); + + if (operation_int != DIFF_OPERATION_REMOVE) { + // Print comma between path and value + builder.append(", "); + // Read value length + long value_length = buffer.getPackedLong(); + + Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), + buffer, + value_length - 1, + charsetName); + buffer.forward((int) value_length - 1); + // Read value + if (jsonValue.m_type == Json_enum_type.ERROR) { + throw new IllegalArgumentException("parsing json value"); + } + StringBuilder jsonBuilder = new StringBuilder(); + jsonValue.toJsonString(jsonBuilder, charsetName); + builder.append(jsonBuilder); + } + + // Print closing parenthesis + if (!buffer.hasRemaining() || operation_names.get(diff_i + 1) != operation_names.get(diff_i)) { + builder.append(")"); + } + + if (buffer.hasRemaining()) { + builder.append(", "); + } + diff_i++; + } + + return builder; + } + + private static String json_diff_operation_name(int operationType, int last_path_char) { + switch (operationType) { + case DIFF_OPERATION_REPLACE: + return "JSON_REPLACE"; + case DIFF_OPERATION_INSERT: + if (last_path_char == ']') { + return "JSON_ARRAY_INSERT"; + } else { + return "JSON_INSERT"; + } + case DIFF_OPERATION_REMOVE: + return "JSON_REMOVE"; + } + throw new IllegalArgumentException("illeagal operationType : " + operationType); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java index be61c99f..408eaa4e 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java @@ -1484,7 +1484,9 @@ public class LogBuffer { for (int bit = 0; bit < len; bit += 8) { int flag = ((int) buf[pos++]) & 0xff; - if (flag == 0) continue; + if (flag == 0) { + continue; + } if ((flag & 0x01) != 0) bitmap.set(bit); if ((flag & 0x02) != 0) bitmap.set(bit + 1); if ((flag & 0x04) != 0) bitmap.set(bit + 2); diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java index 639e9875..41739cf8 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java @@ -372,6 +372,14 @@ public final class LogDecoder { header.putGtid(context.getGtidSet(), gtidLogEvent); return event; } + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: { + RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent, true); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + header.putGtid(context.getGtidSet(), gtidLogEvent); + return event; + } case LogEvent.GTID_LOG_EVENT: case LogEvent.ANONYMOUS_GTID_LOG_EVENT: { GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent); diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java index 8a5957d0..41208516 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java @@ -169,9 +169,15 @@ public abstract class LogEvent { /* Prepared XA transaction terminal event similar to Xid */ public static final int XA_PREPARE_LOG_EVENT = 38; + /** + * Extension of UPDATE_ROWS_EVENT, allowing partial values according to + * binlog_row_value_options. + */ + public static final int PARTIAL_UPDATE_ROWS_EVENT = 39; + // mariaDb 5.5.34 /* New MySQL/Sun events are to be added right above this comment */ - public static final int MYSQL_EVENTS_END = 39; + public static final int MYSQL_EVENTS_END = 49; public static final int MARIA_EVENTS_BEGIN = 160; /* New Maria event numbers start from here */ @@ -361,6 +367,8 @@ public abstract class LogEvent { return "Anonymous_Gtid"; case PREVIOUS_GTIDS_LOG_EVENT: return "Previous_gtids"; + case PARTIAL_UPDATE_ROWS_EVENT: + return "Update_rows_partial"; default: return "Unknown"; /* impossible */ } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java index 8492e2d7..a14b9111 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java @@ -211,6 +211,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 { postHeaderLen[TRANSACTION_CONTEXT_EVENT - 1] = TRANSACTION_CONTEXT_HEADER_LEN; postHeaderLen[VIEW_CHANGE_EVENT - 1] = VIEW_CHANGE_HEADER_LEN; postHeaderLen[XA_PREPARE_LOG_EVENT - 1] = XA_PREPARE_HEADER_LEN; + postHeaderLen[PARTIAL_UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2; // mariadb 10 postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN; diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java index f6e0459c..c732da99 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java @@ -1,6 +1,7 @@ package com.taobao.tddl.dbsync.binlog.event; import java.io.IOException; +import java.math.BigInteger; import java.nio.charset.Charset; import com.taobao.tddl.dbsync.binlog.CharsetConversion; @@ -285,12 +286,12 @@ public class QueryLogEvent extends LogEvent { * type, * sql_mode */ - + 1 + 1 + 255 /* - * type, - * length - * , - * catalog - */ + + 1 + 1 + 255/* + * type, + * length + * , + * catalog + */ + 1 + 4 /* * type, * auto_increment @@ -330,14 +331,32 @@ public class QueryLogEvent extends LogEvent { * MariaDb type, * sec_part of NOW() */ - + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 32 * 3 - + 1 + 60/* - * type , - * user_len , - * user , - * host_len , - * host - */); + + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 /* + * type + * , + * microseconds + */+ 1 + 32 + * 3 + 1 + 60/* + * type , + * user_len + * , user , + * host_len + * , host + */) + + 1 + 1 /* + * type, + * explicit_def + * ..ts + */+ 1 + 8 /* + * type, + * xid + * of + * DDL + */+ 1 + 2 /* + * type + * , + * default_collation_for_utf8mb4_number + */+ 1 /* sql_require_primary_key */; /** * Fixed data part: *
    @@ -394,7 +413,7 @@ public class QueryLogEvent extends LogEvent { // inspection by the DBA private final long execTime; private final int errorCode; - private final long sessionId; /* thread_id */ + private final long sessionId; /* thread_id */ /** * 'flags2' is a second set of flags (on top of those in Log_event), for @@ -412,6 +431,8 @@ public class QueryLogEvent extends LogEvent { private int clientCharset = -1; private int clientCollation = -1; private int serverCollation = -1; + private int tvSec = -1; + private BigInteger ddlXid = BigInteger.valueOf(-1L); private String charsetName; private String timezone; @@ -562,6 +583,21 @@ public class QueryLogEvent extends LogEvent { */ public static final int Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP = 16; + /** + * The variable carries xid info of 2pc-aware (recoverable) DDL queries. + */ + public static final int Q_DDL_LOGGED_WITH_XID = 17; + /** + * This variable stores the default collation for the utf8mb4 character set. + * Used to support cross-version replication. + */ + public static final int Q_DEFAULT_COLLATION_FOR_UTF8MB4 = 18; + + /** + * Replicate sql_require_primary_key. + */ + public static final int Q_SQL_REQUIRE_PRIMARY_KEY = 19; + /** * FROM MariaDB 5.5.34 */ @@ -625,7 +661,7 @@ public class QueryLogEvent extends LogEvent { break; case Q_MICROSECONDS: // when.tv_usec= uint3korr(pos); - buffer.forward(3); + tvSec = buffer.getInt24(); break; case Q_UPDATED_DB_NAMES: int mtsAccessedDbs = buffer.getUint8(); @@ -646,6 +682,19 @@ public class QueryLogEvent extends LogEvent { } break; case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP: + // thd->variables.explicit_defaults_for_timestamp + buffer.forward(1); + break; + case Q_DDL_LOGGED_WITH_XID: + ddlXid = buffer.getUlong64(); + break; + case Q_DEFAULT_COLLATION_FOR_UTF8MB4: + // int2store(start, + // default_collation_for_utf8mb4_number); + buffer.forward(2); + break; + case Q_SQL_REQUIRE_PRIMARY_KEY: + // *start++ = thd->variables.sql_require_primary_key; buffer.forward(1); break; case Q_HRNOW: @@ -657,8 +706,10 @@ public class QueryLogEvent extends LogEvent { * That's why you must write status vars in growing * order of code */ - if (logger.isDebugEnabled()) logger.debug("Query_log_event has unknown status vars (first has code: " - + code + "), skipping the rest of them"); + if (logger.isDebugEnabled()) { + logger.debug("Query_log_event has unknown status vars (first has code: " + code + + "), skipping the rest of them"); + } break; // Break loop } } @@ -695,6 +746,12 @@ public class QueryLogEvent extends LogEvent { return "Q_UPDATED_DB_NAMES"; case Q_MICROSECONDS: return "Q_MICROSECONDS"; + case Q_DDL_LOGGED_WITH_XID: + return "Q_DDL_LOGGED_WITH_XID"; + case Q_DEFAULT_COLLATION_FOR_UTF8MB4: + return "Q_DEFAULT_COLLATION_FOR_UTF8MB4"; + case Q_SQL_REQUIRE_PRIMARY_KEY: + return "Q_SQL_REQUIRE_PRIMARY_KEY"; } return "CODE#" + code; } @@ -777,6 +834,14 @@ public class QueryLogEvent extends LogEvent { return serverCollation; } + public int getTvSec() { + return tvSec; + } + + public BigInteger getDdlXid() { + return ddlXid; + } + /** * Returns the sql_mode value. *

    diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java index 8cd57332..013494d5 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java @@ -10,6 +10,7 @@ import org.apache.commons.logging.LogFactory; import com.taobao.tddl.dbsync.binlog.JsonConversion; import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value; +import com.taobao.tddl.dbsync.binlog.JsonDiffConversion; import com.taobao.tddl.dbsync.binlog.LogBuffer; import com.taobao.tddl.dbsync.binlog.LogEvent; @@ -31,22 +32,33 @@ public final class RowsLogBuffer { private final LogBuffer buffer; private final int columnLen; + private final int jsonColumnCount; private final String charsetName; - // private Calendar cal; private final BitSet nullBits; private int nullBitIndex; + // Read value_options if this is AI for PARTIAL_UPDATE_ROWS_EVENT + private final boolean partial; + private final BitSet partialBits; + private boolean fNull; private int javaType; private int length; private Serializable value; - public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName){ + public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName, int jsonColumnCount, boolean partial){ this.buffer = buffer; this.columnLen = columnLen; this.charsetName = charsetName; + this.partial = partial; + this.jsonColumnCount = jsonColumnCount; this.nullBits = new BitSet(columnLen); + this.partialBits = new BitSet(1); + } + + public final boolean nextOneRow(BitSet columns) { + return nextOneRow(columns, false); } /** @@ -55,18 +67,30 @@ public final class RowsLogBuffer { * @see mysql-5.1.60/sql/log_event.cc - * Rows_log_event::print_verbose_one_row */ - public final boolean nextOneRow(BitSet columns) { + public final boolean nextOneRow(BitSet columns, boolean after) { final boolean hasOneRow = buffer.hasRemaining(); if (hasOneRow) { int column = 0; for (int i = 0; i < columnLen; i++) - if (columns.get(i)) column++; + if (columns.get(i)) { + column++; + } + if (after && partial) { + partialBits.clear(); + long valueOptions = buffer.getPackedLong(); + int PARTIAL_JSON_UPDATES = 1; + if ((valueOptions & PARTIAL_JSON_UPDATES) != 0) { + partialBits.set(1); + buffer.forward((jsonColumnCount + 7) / 8); + } + } nullBitIndex = 0; nullBits.clear(); buffer.fillBitmap(nullBits, column); + } return hasOneRow; } @@ -77,8 +101,8 @@ public final class RowsLogBuffer { * @see mysql-5.1.60/sql/log_event.cc - * Rows_log_event::print_verbose_one_row */ - public final Serializable nextValue(final int type, final int meta) { - return nextValue(type, meta, false); + public final Serializable nextValue(final String columName, final int columnIndex, final int type, final int meta) { + return nextValue(columName, columnIndex, type, meta, false); } /** @@ -87,7 +111,8 @@ public final class RowsLogBuffer { * @see mysql-5.1.60/sql/log_event.cc - * Rows_log_event::print_verbose_one_row */ - public final Serializable nextValue(final int type, final int meta, boolean isBinary) { + public final Serializable nextValue(final String columName, final int columnIndex, final int type, final int meta, + boolean isBinary) { fNull = nullBits.get(nullBitIndex++); if (fNull) { @@ -97,7 +122,7 @@ public final class RowsLogBuffer { return null; } else { // Extracting field value from packed buffer. - return fetchValue(type, meta, isBinary); + return fetchValue(columName, columnIndex, type, meta, isBinary); } } @@ -248,7 +273,7 @@ public final class RowsLogBuffer { * * @see mysql-5.1.60/sql/log_event.cc - log_event_print_value */ - final Serializable fetchValue(int type, final int meta, boolean isBinary) { + final Serializable fetchValue(String columnName, int columnIndex, int type, final int meta, boolean isBinary) { int len = 0; if (type == LogEvent.MYSQL_TYPE_STRING) { @@ -610,7 +635,7 @@ public final class RowsLogBuffer { // (u32 % 10000) / 100, // u32 % 100); - StringBuilder builder = new StringBuilder(12); + StringBuilder builder = new StringBuilder(17); if (i32 < 0) { builder.append('-'); } @@ -1041,17 +1066,34 @@ public final class RowsLogBuffer { default: throw new IllegalArgumentException("!! Unknown JSON packlen = " + meta); } - if (0 == len) { - // fixed issue #1 by lava, json column of zero length has no - // value, value parsing should be skipped - value = ""; - } else { + + if (partialBits.get(1)) { + // print_json_diff int position = buffer.position(); - Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName); - StringBuilder builder = new StringBuilder(); - jsonValue.toJsonString(builder, charsetName); + StringBuilder builder = JsonDiffConversion.print_json_diff(buffer, + len, + columnName, + columnIndex, + charsetName); value = builder.toString(); buffer.position(position + len); + } else { + if (0 == len) { + // fixed issue #1 by lava, json column of zero length + // has no + // value, value parsing should be skipped + value = ""; + } else { + int position = buffer.position(); + Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), + buffer, + len - 1, + charsetName); + StringBuilder builder = new StringBuilder(); + jsonValue.toJsonString(builder, charsetName); + value = builder.toString(); + buffer.position(position + len); + } } javaType = Types.VARCHAR; length = len; @@ -1120,7 +1162,7 @@ public final class RowsLogBuffer { return length; } - private String usecondsToStr(int frac, int meta) { + public static String usecondsToStr(int frac, int meta) { String sec = String.valueOf(frac); if (meta > 6) { throw new IllegalArgumentException("unknow useconds meta : " + meta); @@ -1139,7 +1181,7 @@ public final class RowsLogBuffer { return sec.substring(0, meta); } - private void appendNumber4(StringBuilder builder, int d) { + public static void appendNumber4(StringBuilder builder, int d) { if (d >= 1000) { builder.append(digits[d / 1000]) .append(digits[(d / 100) % 10]) @@ -1151,7 +1193,7 @@ public final class RowsLogBuffer { } } - private void appendNumber3(StringBuilder builder, int d) { + public static void appendNumber3(StringBuilder builder, int d) { if (d >= 100) { builder.append(digits[d / 100]).append(digits[(d / 10) % 10]).append(digits[d % 10]); } else { @@ -1160,7 +1202,7 @@ public final class RowsLogBuffer { } } - private void appendNumber2(StringBuilder builder, int d) { + public static void appendNumber2(StringBuilder builder, int d) { if (d >= 10) { builder.append(digits[(d / 10) % 10]).append(digits[d % 10]); } else { diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java index f2ce79b0..f11b0734 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java @@ -5,6 +5,7 @@ import java.util.BitSet; import com.taobao.tddl.dbsync.binlog.LogBuffer; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo; /** * Common base class for all row-containing log events. @@ -61,6 +62,7 @@ public abstract class RowsLogEvent extends LogEvent { /** Bitmap denoting columns available */ protected final int columnLen; + protected final boolean partial; protected final BitSet columns; /** @@ -71,6 +73,8 @@ public abstract class RowsLogEvent extends LogEvent { */ protected final BitSet changeColumns; + protected int jsonColumnCount = 0; + /** XXX: Don't handle buffer in another thread. */ private final LogBuffer rowsBuf; /* * The rows in @@ -109,6 +113,10 @@ public abstract class RowsLogEvent extends LogEvent { public static final int RW_V_EXTRAINFO_TAG = 0; public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + this(header, buffer, descriptionEvent, false); + } + + public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, boolean partial){ super(header); final int commonHeaderLen = descriptionEvent.commonHeaderLen; @@ -153,9 +161,11 @@ public abstract class RowsLogEvent extends LogEvent { buffer.position(commonHeaderLen + postHeaderLen + headerLen); columnLen = (int) buffer.getPackedLong(); + this.partial = partial; columns = buffer.getBitmap(columnLen); - if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) { + if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT + || header.type == PARTIAL_UPDATE_ROWS_EVENT) { changeColumns = buffer.getBitmap(columnLen); } else { changeColumns = columns; @@ -175,6 +185,17 @@ public abstract class RowsLogEvent extends LogEvent { // delete original table map events stored in the map). context.clearAllTables(); } + + int jsonColumnCount = 0; + int columnCnt = table.getColumnCnt(); + ColumnInfo[] columnInfo = table.getColumnInfo(); + for (int i = 0; i < columnCnt; i++) { + ColumnInfo info = columnInfo[i]; + if (info.type == LogEvent.MYSQL_TYPE_JSON) { + jsonColumnCount++; + } + } + this.jsonColumnCount = jsonColumnCount; } public final long getTableId() { @@ -194,7 +215,7 @@ public abstract class RowsLogEvent extends LogEvent { } public final RowsLogBuffer getRowsBuf(String charsetName) { - return new RowsLogBuffer(rowsBuf, columnLen, charsetName); + return new RowsLogBuffer(rowsBuf, columnLen, charsetName, jsonColumnCount, partial); } public final int getFlags(final int flags) { diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java index e6550b27..18201ea9 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java @@ -1,6 +1,8 @@ package com.taobao.tddl.dbsync.binlog.event; +import java.util.ArrayList; import java.util.BitSet; +import java.util.List; import com.taobao.tddl.dbsync.binlog.LogBuffer; import com.taobao.tddl.dbsync.binlog.LogEvent; @@ -88,6 +90,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent; * first byte, the ninth is in the least significant bit of the second byte, and * so on. * + * + * optional metadata fields + * optional metadata fields are stored in Type, Length, Value(TLV) format. + * Type takes 1 byte. Length is a packed integer value. Values takes Length + * bytes. + * There are some optional metadata defined. They are listed in the table + * + * @ref Table_table_map_event_optional_metadata. Optional metadata fields follow + * null_bits. Whether binlogging an optional metadata is decided by the server. + * The order is not defined, so they can be binlogged in any order. + * * * The table below lists all column types, along with the numerical identifier * for it and the size and interpretation of meta-data used to describe the @@ -284,7 +297,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent; * of the geometry: 1, 2, 3, or 4. * * - * * @author Changyuan.lh * @version 1.0 */ @@ -322,20 +334,63 @@ public final class TableMapLogEvent extends LogEvent { */ public static final class ColumnInfo { - public int type; - public int meta; + public int type; + public int meta; + public String name; + public boolean unsigned; + public boolean pk; + public List set_enum_values; + public int charset; // 可以通过CharsetUtil进行转化 + public int geoType; + public boolean nullable; + + @Override + public String toString() { + return "ColumnInfo [type=" + type + ", meta=" + meta + ", name=" + name + ", unsigned=" + unsigned + + ", pk=" + pk + ", set_enum_values=" + set_enum_values + ", charset=" + charset + ", geoType=" + + geoType + ", nullable=" + nullable + "]"; + } } protected final int columnCnt; - protected final ColumnInfo[] columnInfo; // buffer for field - // metadata + protected final ColumnInfo[] columnInfo; // buffer + // for + // field + // metadata protected final long tableId; protected BitSet nullBits; /** TM = "Table Map" */ - public static final int TM_MAPID_OFFSET = 0; - public static final int TM_FLAGS_OFFSET = 6; + public static final int TM_MAPID_OFFSET = 0; + public static final int TM_FLAGS_OFFSET = 6; + + // UNSIGNED flag of numeric columns + public static final int SIGNEDNESS = 1; + // Default character set of string columns + public static final int DEFAULT_CHARSET = 2; + // Character set of string columns + public static final int COLUMN_CHARSET = 3; + public static final int COLUMN_NAME = 4; + // String value of SET columns + public static final int SET_STR_VALUE = 5; + // String value of ENUM columns + public static final int ENUM_STR_VALUE = 6; + // Real type of geometry columns + public static final int GEOMETRY_TYPE = 7; + // Primary key without prefix + public static final int SIMPLE_PRIMARY_KEY = 8; + // Primary key with prefix + public static final int PRIMARY_KEY_WITH_PREFIX = 9; + + private int default_charset; + private boolean existOptionalMetaData = false; + + private static final class Pair { + + public int col_index; + public int col_charset; + } /** * Constructor used by slave to read the event from the binary log. @@ -379,7 +434,91 @@ public final class TableMapLogEvent extends LogEvent { final int fieldSize = (int) buffer.getPackedLong(); decodeFields(buffer, fieldSize); nullBits = buffer.getBitmap(columnCnt); + + for (int i = 0; i < columnCnt; i++) { + if (nullBits.get(i)) { + columnInfo[i].nullable = true; + } + } + /* + * After null_bits field, there are some new fields for extra + * metadata. + */ + existOptionalMetaData = false; + List defaultCharsetPairs = null; + List columnCharsets = null; + while (buffer.hasRemaining()) { + // optional metadata fields + int type = buffer.getUint8(); + int len = (int) buffer.getPackedLong(); + + switch (type) { + case SIGNEDNESS: + parse_signedness(buffer, len); + break; + case DEFAULT_CHARSET: + defaultCharsetPairs = parse_default_charset(buffer, len); + break; + case COLUMN_CHARSET: + columnCharsets = parse_column_charset(buffer, len); + break; + case COLUMN_NAME: + // set @@global.binlog_row_metadata='FULL' + // 主要是补充列名相关信息 + existOptionalMetaData = true; + parse_column_name(buffer, len); + break; + case SET_STR_VALUE: + parse_set_str_value(buffer, len, true); + break; + case ENUM_STR_VALUE: + parse_set_str_value(buffer, len, false); + break; + case GEOMETRY_TYPE: + parse_geometry_type(buffer, len); + break; + case SIMPLE_PRIMARY_KEY: + parse_simple_pk(buffer, len); + break; + case PRIMARY_KEY_WITH_PREFIX: + parse_pk_with_prefix(buffer, len); + break; + default: + throw new IllegalArgumentException("unknow type : " + type); + } + } + + if (existOptionalMetaData) { + int index = 0; + int char_col_index = 0; + for (int i = 0; i < columnCnt; i++) { + int cs = -1; + int type = getRealType(columnInfo[i].type, columnInfo[i].meta); + if (is_character_type(type)) { + if (defaultCharsetPairs != null && !defaultCharsetPairs.isEmpty()) { + if (index < defaultCharsetPairs.size() + && char_col_index == defaultCharsetPairs.get(index).col_index) { + cs = defaultCharsetPairs.get(index).col_charset; + index++; + } else { + cs = default_charset; + } + + char_col_index++; + } else if (columnCharsets != null) { + cs = columnCharsets.get(index); + index++; + } + + columnInfo[i].charset = cs; + } + } + } } + + // for (int i = 0; i < columnCnt; i++) { + // System.out.println(columnInfo[i]); + // } } /** @@ -459,6 +598,192 @@ public final class TableMapLogEvent extends LogEvent { buffer.limit(limit); } + private void parse_signedness(LogBuffer buffer, int length) { + // stores the signedness flags extracted from field + List datas = new ArrayList(); + for (int i = 0; i < length; i++) { + int ut = buffer.getUint8(); + for (int c = 0x80; c != 0; c >>= 1) { + datas.add((ut & c) > 0); + } + } + + int index = 0; + for (int i = 0; i < columnCnt; i++) { + if (is_numeric_type(columnInfo[i].type)) { + columnInfo[i].unsigned = datas.get(index); + index++; + } + } + } + + private List parse_default_charset(LogBuffer buffer, int length) { + // stores collation numbers extracted from field. + int limit = buffer.position() + length; + this.default_charset = (int) buffer.getPackedLong(); + List datas = new ArrayList(); + while (buffer.hasRemaining() && buffer.position() < limit) { + int col_index = (int) buffer.getPackedLong(); + int col_charset = (int) buffer.getPackedLong(); + + Pair pair = new Pair(); + pair.col_index = col_index; + pair.col_charset = col_charset; + datas.add(pair); + } + + return datas; + } + + private List parse_column_charset(LogBuffer buffer, int length) { + // stores collation numbers extracted from field. + int limit = buffer.position() + length; + List datas = new ArrayList(); + while (buffer.hasRemaining() && buffer.position() < limit) { + int col_charset = (int) buffer.getPackedLong(); + datas.add(col_charset); + } + + return datas; + } + + private void parse_column_name(LogBuffer buffer, int length) { + // stores column names extracted from field + int limit = buffer.position() + length; + int index = 0; + while (buffer.hasRemaining() && buffer.position() < limit) { + int len = (int) buffer.getPackedLong(); + columnInfo[index++].name = buffer.getFixString(len); + } + } + + private void parse_set_str_value(LogBuffer buffer, int length, boolean set) { + // stores SET/ENUM column's string values extracted from + // field. Each SET/ENUM column's string values are stored + // into a string separate vector. All of them are stored + // in 'vec'. + int limit = buffer.position() + length; + List> datas = new ArrayList>(); + while (buffer.hasRemaining() && buffer.position() < limit) { + int count = (int) buffer.getPackedLong(); + List data = new ArrayList(count); + for (int i = 0; i < count; i++) { + int len1 = (int) buffer.getPackedLong(); + data.add(buffer.getFixString(len1)); + } + + datas.add(data); + } + + int index = 0; + for (int i = 0; i < columnCnt; i++) { + if (set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_SET) { + columnInfo[i].set_enum_values = datas.get(index); + index++; + } + + if (!set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_ENUM) { + columnInfo[i].set_enum_values = datas.get(index); + index++; + } + } + } + + private void parse_geometry_type(LogBuffer buffer, int length) { + // stores geometry column's types extracted from field. + int limit = buffer.position() + length; + + List datas = new ArrayList(); + while (buffer.hasRemaining() && buffer.position() < limit) { + int col_type = (int) buffer.getPackedLong(); + datas.add(col_type); + } + + int index = 0; + for (int i = 0; i < columnCnt; i++) { + if (columnInfo[i].type == LogEvent.MYSQL_TYPE_GEOMETRY) { + columnInfo[i].geoType = datas.get(index); + index++; + } + } + } + + private void parse_simple_pk(LogBuffer buffer, int length) { + // stores primary key's column information extracted from + // field. Each column has an index and a prefix which are + // stored as a unit_pair. prefix is always 0 for + // SIMPLE_PRIMARY_KEY field. + + int limit = buffer.position() + length; + while (buffer.hasRemaining() && buffer.position() < limit) { + int col_index = (int) buffer.getPackedLong(); + columnInfo[col_index].pk = true; + } + } + + private void parse_pk_with_prefix(LogBuffer buffer, int length) { + // stores primary key's column information extracted from + // field. Each column has an index and a prefix which are + // stored as a unit_pair. + int limit = buffer.position() + length; + while (buffer.hasRemaining() && buffer.position() < limit) { + int col_index = (int) buffer.getPackedLong(); + // prefix length, 比如 char(32) + @SuppressWarnings("unused") + int col_prefix = (int) buffer.getPackedLong(); + columnInfo[col_index].pk = true; + } + } + + private boolean is_numeric_type(int type) { + switch (type) { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_NEWDECIMAL: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + return true; + default: + return false; + } + } + + private boolean is_character_type(int type) { + switch (type) { + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_BLOB: + return true; + default: + return false; + } + } + + private int getRealType(int type, int meta) { + if (type == LogEvent.MYSQL_TYPE_STRING) { + if (meta >= 256) { + int byte0 = meta >> 8; + if ((byte0 & 0x30) != 0x30) { + /* a long CHAR() field: see #37426 */ + type = byte0 | 0x30; + } else { + switch (byte0) { + case LogEvent.MYSQL_TYPE_SET: + case LogEvent.MYSQL_TYPE_ENUM: + case LogEvent.MYSQL_TYPE_STRING: + type = byte0; + } + } + } + } + + return type; + } + public final String getDbName() { return dbname; } @@ -478,4 +803,13 @@ public final class TableMapLogEvent extends LogEvent { public final long getTableId() { return tableId; } + + public boolean isExistOptionalMetaData() { + return existOptionalMetaData; + } + + public void setExistOptionalMetaData(boolean existOptional) { + this.existOptionalMetaData = existOptional; + } + } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java index f5e5866e..b604e04d 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java @@ -14,6 +14,11 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer; public final class UpdateRowsLogEvent extends RowsLogEvent { public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ - super(header, buffer, descriptionEvent); + super(header, buffer, descriptionEvent, false); + } + + public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, + boolean partial){ + super(header, buffer, descriptionEvent, partial); } } diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java index c48082df..3648da68 100644 --- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java @@ -45,6 +45,7 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest { parseRowsEvent((WriteRowsLogEvent) event); break; case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: case LogEvent.UPDATE_ROWS_EVENT: parseRowsEvent((UpdateRowsLogEvent) event); break; diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java index 7673f71b..aada3de8 100644 --- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java @@ -53,6 +53,7 @@ public class FileLogFetcherTest extends BaseLogFetcherTest { parseRowsEvent((WriteRowsLogEvent) event); break; case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: case LogEvent.UPDATE_ROWS_EVENT: parseRowsEvent((UpdateRowsLogEvent) event); break; diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java new file mode 100644 index 00000000..ba98a87f --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java @@ -0,0 +1,258 @@ +package com.alibaba.otter.canal.parse.driver.mysql.utils; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; + +/** + * mysql collation转换mapping关系表 + * + * @author agapple 2018年11月5日 下午1:01:15 + * @since 1.1.2 + */ +public class CharsetUtil { + + private static final String[] INDEX_TO_CHARSET = new String[2048]; + private static final Map CHARSET_TO_INDEX = new HashMap(); + static { + INDEX_TO_CHARSET[1] = "big5"; + INDEX_TO_CHARSET[84] = "big5"; + + INDEX_TO_CHARSET[3] = "dec8"; + INDEX_TO_CHARSET[69] = "dec8"; + + INDEX_TO_CHARSET[4] = "cp850"; + INDEX_TO_CHARSET[80] = "cp850"; + + INDEX_TO_CHARSET[6] = "hp8"; + INDEX_TO_CHARSET[72] = "hp8"; + + INDEX_TO_CHARSET[7] = "koi8r"; + INDEX_TO_CHARSET[74] = "koi8r"; + + INDEX_TO_CHARSET[5] = "latin1"; + INDEX_TO_CHARSET[8] = "latin1"; + INDEX_TO_CHARSET[15] = "latin1"; + INDEX_TO_CHARSET[31] = "latin1"; + INDEX_TO_CHARSET[47] = "latin1"; + INDEX_TO_CHARSET[48] = "latin1"; + INDEX_TO_CHARSET[49] = "latin1"; + INDEX_TO_CHARSET[94] = "latin1"; + + INDEX_TO_CHARSET[9] = "latin2"; + INDEX_TO_CHARSET[21] = "latin2"; + INDEX_TO_CHARSET[27] = "latin2"; + INDEX_TO_CHARSET[77] = "latin2"; + + INDEX_TO_CHARSET[10] = "swe7"; + INDEX_TO_CHARSET[82] = "swe7"; + + INDEX_TO_CHARSET[11] = "ascii"; + INDEX_TO_CHARSET[65] = "ascii"; + + INDEX_TO_CHARSET[12] = "ujis"; + INDEX_TO_CHARSET[91] = "ujis"; + + INDEX_TO_CHARSET[13] = "sjis"; + INDEX_TO_CHARSET[88] = "sjis"; + + INDEX_TO_CHARSET[16] = "hebrew"; + INDEX_TO_CHARSET[71] = "hebrew"; + + INDEX_TO_CHARSET[18] = "tis620"; + INDEX_TO_CHARSET[69] = "tis620"; + + INDEX_TO_CHARSET[19] = "euckr"; + INDEX_TO_CHARSET[85] = "euckr"; + + INDEX_TO_CHARSET[22] = "koi8u"; + INDEX_TO_CHARSET[75] = "koi8u"; + + INDEX_TO_CHARSET[24] = "gb2312"; + INDEX_TO_CHARSET[86] = "gb2312"; + + INDEX_TO_CHARSET[25] = "greek"; + INDEX_TO_CHARSET[70] = "greek"; + + INDEX_TO_CHARSET[26] = "cp1250"; + INDEX_TO_CHARSET[34] = "cp1250"; + INDEX_TO_CHARSET[44] = "cp1250"; + INDEX_TO_CHARSET[66] = "cp1250"; + INDEX_TO_CHARSET[99] = "cp1250"; + + INDEX_TO_CHARSET[28] = "gbk"; + INDEX_TO_CHARSET[87] = "gbk"; + + INDEX_TO_CHARSET[30] = "latin5"; + INDEX_TO_CHARSET[78] = "latin5"; + + INDEX_TO_CHARSET[32] = "armscii8"; + INDEX_TO_CHARSET[64] = "armscii8"; + + INDEX_TO_CHARSET[33] = "utf8"; + INDEX_TO_CHARSET[83] = "utf8"; + for (int i = 192; i <= 223; i++) { + INDEX_TO_CHARSET[i] = "utf8"; + } + for (int i = 336; i <= 337; i++) { + INDEX_TO_CHARSET[i] = "utf8"; + } + for (int i = 352; i <= 357; i++) { + INDEX_TO_CHARSET[i] = "utf8"; + } + INDEX_TO_CHARSET[368] = "utf8"; + INDEX_TO_CHARSET[2047] = "utf8"; + + INDEX_TO_CHARSET[35] = "ucs2"; + INDEX_TO_CHARSET[90] = "ucs2"; + for (int i = 128; i <= 151; i++) { + INDEX_TO_CHARSET[i] = "ucs2"; + } + INDEX_TO_CHARSET[159] = "ucs2"; + for (int i = 358; i <= 360; i++) { + INDEX_TO_CHARSET[i] = "ucs2"; + } + + INDEX_TO_CHARSET[36] = "cp866"; + INDEX_TO_CHARSET[68] = "cp866"; + + INDEX_TO_CHARSET[37] = "keybcs2"; + INDEX_TO_CHARSET[73] = "keybcs2"; + + INDEX_TO_CHARSET[38] = "macce"; + INDEX_TO_CHARSET[43] = "macce"; + + INDEX_TO_CHARSET[39] = "macroman"; + INDEX_TO_CHARSET[53] = "macroman"; + + INDEX_TO_CHARSET[40] = "cp852"; + INDEX_TO_CHARSET[81] = "cp852"; + + INDEX_TO_CHARSET[20] = "latin7"; + INDEX_TO_CHARSET[41] = "latin7"; + INDEX_TO_CHARSET[42] = "latin7"; + INDEX_TO_CHARSET[79] = "latin7"; + + INDEX_TO_CHARSET[45] = "utf8mb4"; + INDEX_TO_CHARSET[46] = "utf8mb4"; + for (int i = 224; i <= 247; i++) { + INDEX_TO_CHARSET[i] = "utf8mb4"; + } + for (int i = 255; i <= 271; i++) { + INDEX_TO_CHARSET[i] = "utf8mb4"; + } + for (int i = 273; i <= 275; i++) { + INDEX_TO_CHARSET[i] = "utf8mb4"; + } + for (int i = 277; i <= 294; i++) { + INDEX_TO_CHARSET[i] = "utf8mb4"; + } + for (int i = 296; i <= 298; i++) { + INDEX_TO_CHARSET[i] = "utf8mb4"; + } + INDEX_TO_CHARSET[300] = "utf8mb4"; + for (int i = 303; i <= 307; i++) { + INDEX_TO_CHARSET[i] = "utf8mb4"; + } + INDEX_TO_CHARSET[326] = "utf8mb4"; + INDEX_TO_CHARSET[328] = "utf8mb4"; + + INDEX_TO_CHARSET[14] = "cp1251"; + INDEX_TO_CHARSET[23] = "cp1251"; + INDEX_TO_CHARSET[50] = "cp1251"; + INDEX_TO_CHARSET[51] = "cp1251"; + INDEX_TO_CHARSET[52] = "cp1251"; + + INDEX_TO_CHARSET[54] = "utf16"; + INDEX_TO_CHARSET[55] = "utf16"; + for (int i = 101; i <= 124; i++) { + INDEX_TO_CHARSET[i] = "utf16"; + } + INDEX_TO_CHARSET[327] = "utf16"; + + INDEX_TO_CHARSET[56] = "utf16le"; + INDEX_TO_CHARSET[62] = "utf16le"; + + INDEX_TO_CHARSET[57] = "cp1256"; + INDEX_TO_CHARSET[67] = "cp1256"; + + INDEX_TO_CHARSET[29] = "cp1257"; + INDEX_TO_CHARSET[58] = "cp1257"; + INDEX_TO_CHARSET[59] = "cp1257"; + + INDEX_TO_CHARSET[60] = "utf32"; + INDEX_TO_CHARSET[61] = "utf32"; + for (int i = 160; i <= 183; i++) { + INDEX_TO_CHARSET[i] = "utf32"; + } + INDEX_TO_CHARSET[391] = "utf32"; + + INDEX_TO_CHARSET[63] = "binary"; + + INDEX_TO_CHARSET[92] = "geostd8"; + INDEX_TO_CHARSET[93] = "geostd8"; + + INDEX_TO_CHARSET[95] = "cp932"; + INDEX_TO_CHARSET[96] = "cp932"; + + INDEX_TO_CHARSET[97] = "eucjpms"; + INDEX_TO_CHARSET[98] = "eucjpms"; + + for (int i = 248; i <= 250; i++) { + INDEX_TO_CHARSET[i] = "gb18030"; + } + + // charset --> index + for (int i = 0; i < 2048; i++) { + String charset = INDEX_TO_CHARSET[i]; + if (charset != null && CHARSET_TO_INDEX.get(charset) == null) { + CHARSET_TO_INDEX.put(charset, i); + } + } + CHARSET_TO_INDEX.put("iso-8859-1", 14); + CHARSET_TO_INDEX.put("iso_8859_1", 14); + CHARSET_TO_INDEX.put("utf-8", 33); + CHARSET_TO_INDEX.put("utf8mb4", 45); + } + + public static final String getCharset(int index) { + return INDEX_TO_CHARSET[index]; + } + + public static final int getIndex(String charset) { + if (charset == null || charset.length() == 0) { + return 0; + } else { + Integer i = CHARSET_TO_INDEX.get(charset.toLowerCase()); + return (i == null) ? 0 : i.intValue(); + } + } + + /** + * 'utf8' COLLATE 'utf8_general_ci' + * + * @param charset + * @return + */ + public static final String collateCharset(String charset) { + String[] output = StringUtils.split(charset, "COLLATE"); + return output[0].replace('\'', ' ').trim(); + } + + public static String getJavaCharset(String charset) { + if ("utf8".equals(charset)) { + return charset; + } + + if (StringUtils.endsWithIgnoreCase(charset, "utf8mb4")) { + return "utf-8"; + } + + if (StringUtils.endsWithIgnoreCase(charset, "binary")) { + return "iso_8859_1"; + } + + return charset; + } +} diff --git a/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java new file mode 100644 index 00000000..09837d09 --- /dev/null +++ b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java @@ -0,0 +1,51 @@ +package com.alibaba.otter.canal.parse.driver.mysql; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.otter.canal.parse.driver.mysql.utils.CharsetUtil; + +public class CharsetUtilTest { + + @Test + public void testLatin1() { + int charsetIndex = 5; + String charset = "latin1"; + Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex))); + } + + @Test + public void testGbk() { + int charsetIndex = 87; + String charset = "gbk"; + Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex))); + } + + @Test + public void testGb2312() { + int charsetIndex = 24; + String charset = "gb2312"; + Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex))); + } + + @Test + public void testUtf8() { + int charsetIndex = 213; + String charset = "utf8"; + Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex))); + } + + @Test + public void testUtf8mb4() { + int charsetIndex = 235; + String charset = "utf8mb4"; + Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex))); + } + + @Test + public void testBinary() { + int charsetIndex = 63; + String charset = "binary"; + Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex))); + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 8b682f5d..703de818 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -275,6 +275,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement needDmlParse = true; break; case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: case LogEvent.UPDATE_ROWS_EVENT: tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent); needDmlParse = true; diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 0b88af14..5c5432c6 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -120,6 +120,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar case LogEvent.WRITE_ROWS_EVENT: return parseRowsEvent((WriteRowsLogEvent) logEvent); case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: case LogEvent.UPDATE_ROWS_EVENT: return parseRowsEvent((UpdateRowsLogEvent) logEvent); case LogEvent.DELETE_ROWS_EVENT_V1: @@ -261,7 +262,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } boolean isDml = (type == EventType.INSERT || type == EventType.UPDATE || type == EventType.DELETE); - + if (!isSeek && !isDml) { // 使用新的表结构元数据管理方式 EntryPosition position = createPosition(event.getHeader()); @@ -504,7 +505,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar int type = event.getHeader().getType(); if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) { eventType = EventType.INSERT; - } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) { + } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type + || LogEvent.PARTIAL_UPDATE_ROWS_EVENT == type) { eventType = EventType.UPDATE; } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) { eventType = EventType.DELETE; @@ -523,7 +525,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar boolean tableError = false; int rowsCount = 0; - while (buffer.nextOneRow(columns)) { + while (buffer.nextOneRow(columns, false)) { // 处理row记录 RowData.Builder rowDataBuilder = RowData.newBuilder(); if (EventType.INSERT == eventType) { @@ -535,7 +537,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } else { // update需要处理before/after tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta); - if (!buffer.nextOneRow(changeColumns)) { + if (!buffer.nextOneRow(changeColumns, true)) { rowChangeBuider.addRowDatas(rowDataBuilder.build()); break; } @@ -577,7 +579,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException { int columnCnt = event.getTable().getColumnCnt(); ColumnInfo[] columnInfo = event.getTable().getColumnInfo(); - + // mysql8.0针对set @@global.binlog_row_metadata='FULL' 可以记录部分的metadata信息 + boolean existOptionalMetaData = event.getTable().isExistOptionalMetaData(); boolean tableError = false; // check table fileds count，只能处理加字段 boolean existRDSNoPrimaryKey = false; @@ -631,22 +634,43 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar continue; } - if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) { - // 不解析最后一列 - buffer.nextValue(info.type, info.meta, false); - continue; - } - - Column.Builder columnBuilder = Column.newBuilder(); - FieldMeta fieldMeta = null; if (tableMeta != null && !tableError) { // 处理file meta fieldMeta = tableMeta.getFields().get(i); + } + + if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) { + // 不解析最后一列 + buffer.nextValue(fieldMeta.getColumnName(), i, info.type, info.meta, false); + continue; + } + + if (fieldMeta != null && existOptionalMetaData) { + // check column info + boolean check = StringUtils.equalsIgnoreCase(fieldMeta.getColumnName(), info.name); + check &= (fieldMeta.isUnsigned() == info.unsigned); + check &= (fieldMeta.isNullable() == info.nullable); + + if (!check) { + throw new CanalParseException("MySQL8.0 unmatch column metadata & pls submit issue , db : " + + fieldMeta.toString() + " , binlog : " + info.toString() + + " , on : " + event.getHeader().getLogFileName() + ":" + + event.getHeader().getLogPos()); + } + } + + Column.Builder columnBuilder = Column.newBuilder(); + if (fieldMeta != null) { columnBuilder.setName(fieldMeta.getColumnName()); columnBuilder.setIsKey(fieldMeta.isKey()); // 增加mysql type类型,issue 73 columnBuilder.setMysqlType(fieldMeta.getColumnType()); + } else if (existOptionalMetaData) { + columnBuilder.setName(info.name); + columnBuilder.setIsKey(info.pk); + // mysql8.0里没有mysql type类型 + // columnBuilder.setMysqlType(fieldMeta.getColumnType()); } columnBuilder.setIndex(i); columnBuilder.setIsNull(false); @@ -664,12 +688,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar isSingleBit = true; } } - buffer.nextValue(info.type, info.meta, isBinary); - if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) { - // 不解析最后一列 - continue; - } + buffer.nextValue(fieldMeta.getColumnName(), i, info.type, info.meta, isBinary); int javaType = buffer.getJavaType(); if (buffer.isNull()) { columnBuilder.setIsNull(true); @@ -683,7 +703,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar case Types.BIGINT: // 处理unsigned类型 Number number = (Number) value; - if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) { + boolean isUnsigned = (fieldMeta != null ? fieldMeta.isUnsigned() : (existOptionalMetaData ? info.unsigned : false)); + if (isUnsigned && number.longValue() < 0) { switch (buffer.getLength()) { case 1: /* MYSQL_TYPE_TINY */ columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE @@ -772,7 +793,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar default: columnBuilder.setValue(value.toString()); } - } columnBuilder.setSqlType(javaType); diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java index 2cb6ed8b..fd01e6a8 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java @@ -1,7 +1,11 @@ package com.alibaba.otter.canal.parse; import java.io.IOException; +import java.io.Serializable; +import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.util.BitSet; import java.util.List; import org.apache.commons.lang.StringUtils; @@ -24,12 +28,25 @@ import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer; +import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo; +import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.XidLogEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; public class DirectLogFetcherTest { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + protected String binlogFileName = "mysql-bin.000001"; + protected Charset charset = Charset.forName("utf-8"); private boolean isMariaDB; private int binlogChecksum; @@ -37,7 +54,7 @@ public class DirectLogFetcherTest { public void testSimple() { DirectLogFetcher fetcher = new DirectLogFetcher(); try { - MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxx", "xxxx"); + MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"); connector.connect(); updateSettings(connector); loadBinlogChecksum(connector); @@ -67,21 +84,22 @@ public class DirectLogFetcherTest { break; case LogEvent.WRITE_ROWS_EVENT_V1: case LogEvent.WRITE_ROWS_EVENT: - // parseRowsEvent((WriteRowsLogEvent) event); + parseRowsEvent((WriteRowsLogEvent) event); break; case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: case LogEvent.UPDATE_ROWS_EVENT: - // parseRowsEvent((UpdateRowsLogEvent) event); + parseRowsEvent((UpdateRowsLogEvent) event); break; case LogEvent.DELETE_ROWS_EVENT_V1: case LogEvent.DELETE_ROWS_EVENT: - // parseRowsEvent((DeleteRowsLogEvent) event); + parseRowsEvent((DeleteRowsLogEvent) event); break; case LogEvent.QUERY_EVENT: - // parseQueryEvent((QueryLogEvent) event); + parseQueryEvent((QueryLogEvent) event); break; case LogEvent.ROWS_QUERY_LOG_EVENT: - // parseRowsQueryEvent((RowsQueryLogEvent) event); + parseRowsQueryEvent((RowsQueryLogEvent) event); break; case LogEvent.ANNOTATE_ROWS_EVENT: break; @@ -241,4 +259,98 @@ public class DirectLogFetcherTest { exector.update(cmd); } + protected void parseQueryEvent(QueryLogEvent event) { + System.out.println(String.format("================> binlog[%s:%s] , name[%s]", + binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen(), + event.getCatalog())); + System.out.println("sql : " + event.getQuery()); + } + + protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception { + System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader() + .getLogPos() - event.getHeader().getEventLen())); + System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name())); + } + + protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception { + System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader() + .getLogPos() - event.getHeader().getEventLen())); + System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name())); + } + + protected void parseXidEvent(XidLogEvent event) throws Exception { + System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader() + .getLogPos() - event.getHeader().getEventLen())); + System.out.println("xid : " + event.getXid()); + } + + protected void parseRowsEvent(RowsLogEvent event) { + try { + System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", + binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen(), + event.getTable().getDbName(), + event.getTable().getTableName())); + RowsLogBuffer buffer = event.getRowsBuf(charset.name()); + BitSet columns = event.getColumns(); + BitSet changeColumns = event.getChangeColumns(); + while (buffer.nextOneRow(columns)) { + // 处理row记录 + int type = event.getHeader().getType(); + if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) { + // insert的记录放在before字段中 + parseOneRow(event, buffer, columns, true); + } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) { + // delete的记录放在before字段中 + parseOneRow(event, buffer, columns, false); + } else { + // update需要处理before/after + System.out.println("-------> before"); + parseOneRow(event, buffer, columns, false); + if (!buffer.nextOneRow(changeColumns, true)) { + break; + } + System.out.println("-------> after"); + parseOneRow(event, buffer, changeColumns, true); + } + + } + } catch (Exception e) { + throw new RuntimeException("parse row data failed.", e); + } + } + + protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter) + throws UnsupportedEncodingException { + TableMapLogEvent map = event.getTable(); + if (map == null) { + throw new RuntimeException("not found TableMap with tid=" + event.getTableId()); + } + + final int columnCnt = map.getColumnCnt(); + final ColumnInfo[] columnInfo = map.getColumnInfo(); + + for (int i = 0; i < columnCnt; i++) { + if (!cols.get(i)) { + continue; + } + + ColumnInfo info = columnInfo[i]; + buffer.nextValue(null, i, info.type, info.meta); + + if (buffer.isNull()) { + // + } else { + final Serializable value = buffer.getValue(); + if (value instanceof byte[]) { + System.out.println(new String((byte[]) value)); + } else { + System.out.println(value); + } + } + } + + } + } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java index 79b88375..70b2dbd7 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java @@ -89,6 +89,7 @@ public class MysqlBinlogParsePerformanceTest { parseRowsEvent((WriteRowsLogEvent) event, sum); break; case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: case LogEvent.UPDATE_ROWS_EVENT: parseRowsEvent((UpdateRowsLogEvent) event, sum); break; @@ -154,7 +155,7 @@ public class MysqlBinlogParsePerformanceTest { parseOneRow(event, buffer, columns, false); } else { parseOneRow(event, buffer, columns, false); - if (!buffer.nextOneRow(changeColumns)) { + if (!buffer.nextOneRow(changeColumns, true)) { break; } parseOneRow(event, buffer, changeColumns, true); @@ -182,7 +183,7 @@ public class MysqlBinlogParsePerformanceTest { } ColumnInfo info = columnInfo[i]; - buffer.nextValue(info.type, info.meta); + buffer.nextValue(null, i, info.type, info.meta); if (buffer.isNull()) { } else { buffer.getValue(); From 80cf533b050b714ea4a96bef0e7453ab59dd99ee Mon Sep 17 00:00:00 2001 From: mcy Date: Mon, 5 Nov 2018 15:24:32 +0800 Subject: [PATCH 38/42] =?UTF-8?q?=E6=95=B4=E7=90=86=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 4 - .../adapter/es/config/ESSyncConfig.java | 6 ++ .../adapter/es/config/ESSyncConfigLoader.java | 6 ++ .../client/adapter/es/config/SchemaItem.java | 10 ++- .../client/adapter/es/config/SqlParser.java | 21 +++-- .../adapter/es/service/ESEtlService.java | 6 ++ .../adapter/es/service/ESSyncService.java | 26 +++--- .../client/adapter/es/support/ESSyncUtil.java | 24 +++++ .../client/adapter/es/support/ESTemplate.java | 90 ++++++++++++++----- .../client/adapter/hbase/HbaseAdapter.java | 49 +++++----- .../src/main/resources/application.yml | 38 ++++---- 11 files changed, 189 insertions(+), 91 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index 3f11035a..7d10d702 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -12,8 +12,6 @@ import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; @@ -33,8 +31,6 @@ import com.alibaba.otter.canal.client.adapter.support.*; @SPI("es") public class ESAdapter implements OuterAdapter { - private static Logger logger = LoggerFactory.getLogger(ESAdapter.class); - private TransportClient transportClient; private ESSyncService esSyncService; diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java index e327b93f..99d425a6 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -3,6 +3,12 @@ package com.alibaba.otter.canal.client.adapter.es.config; import java.util.ArrayList; import java.util.List; +/** + * ES 映射配置 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class ESSyncConfig { private String dataSourceKey; // 数据源key diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index 7fef12c0..cb1cc274 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -17,6 +17,12 @@ import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +/** + * ES 配置装载器 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class ESSyncConfigLoader { private static Logger logger = LoggerFactory diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java index f092bf78..fc6e076c 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -7,10 +7,16 @@ import java.util.Map; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +/** + * ES 映射配置视图 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class SchemaItem { - private Map aliasTableItems = new LinkedHashMap<>(); - private Map selectFields = new LinkedHashMap<>(); + private Map aliasTableItems = new LinkedHashMap<>(); // 别名对应表名 + private Map selectFields = new LinkedHashMap<>(); // 查询字段 private String sql; private volatile Map> tableItemAliases; diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java index 78fb93ec..0c2d7b2c 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java @@ -34,8 +34,8 @@ public class SqlParser { /** * 解析sql * - * @param sql - * @return + * @param sql sql + * @return 视图对象 */ public static SchemaItem parse(String sql) { try { @@ -67,8 +67,8 @@ public class SqlParser { /** * 归集字段 * - * @param sqlSelectQueryBlock - * @return + * @param sqlSelectQueryBlock sqlSelectQueryBlock + * @return 字段属性列表 */ private static List collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) { return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> { @@ -82,8 +82,8 @@ public class SqlParser { /** * 解析字段 * - * @param expr - * @param fieldItem + * @param expr sql expr + * @param fieldItem 字段属性 */ private static void visitColumn(SQLExpr expr, FieldItem fieldItem) { if (expr instanceof SQLIdentifierExpr) { @@ -123,6 +123,11 @@ public class SqlParser { /** * 解析表 + * + * @param schemaItem 视图对象 + * @param sqlTableSource sqlTableSource + * @param tableItems 表对象列表 + * @param tableItemTmp 表对象(临时) */ private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource, List tableItems, TableItem tableItemTmp) { @@ -174,8 +179,8 @@ public class SqlParser { /** * 解析on条件 * - * @param expr - * @param tableItem + * @param expr sql expr + * @param tableItem 表对象 */ private static void visitOnCondition(SQLExpr expr, TableItem tableItem) { if (!(expr instanceof SQLBinaryOpExpr)) { diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java index d7045e47..51f0a924 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java @@ -31,6 +31,12 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.google.common.base.Joiner; +/** + * ES ETL Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class ESEtlService { private static Logger logger = LoggerFactory.getLogger(ESEtlService.class); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java index a77aaeef..3927c6ab 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java @@ -6,8 +6,6 @@ import java.util.Map; import javax.sql.DataSource; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +21,12 @@ import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.Dml; +/** + * ES 同步 Service + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class ESSyncService { private static Logger logger = LoggerFactory.getLogger(ESSyncService.class); @@ -530,7 +534,7 @@ public class ESSyncService { TableItem tableItem, Map esFieldData) { ESMapping mapping = config.getEsMapping(); - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + Map paramsTmp = new LinkedHashMap<>(); for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { for (FieldItem fieldItem : entry.getValue()) { if (fieldItem.getColumnItems().size() == 1) { @@ -544,7 +548,7 @@ public class ESSyncService { if (fieldName.equals(mapping.get_id())) { fieldName = "_id"; } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + paramsTmp.put(fieldName, value); } } } @@ -555,7 +559,7 @@ public class ESSyncService { dml.getTable(), mapping.get_index()); } - boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData); if (!result) { logger.error("Join table update es index by foreign key error, destination:{}, table: {}, index: {}", config.getDestination(), @@ -625,7 +629,7 @@ public class ESSyncService { } } - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + Map paramsTmp = new LinkedHashMap<>(); for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { for (FieldItem fieldItem : entry.getValue()) { if (fieldItem.getColumnItems().size() == 1) { @@ -638,7 +642,7 @@ public class ESSyncService { if (fieldName.equals(mapping.get_id())) { fieldName = "_id"; } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + paramsTmp.put(fieldName, value); } } } @@ -649,7 +653,7 @@ public class ESSyncService { dml.getTable(), mapping.get_index()); } - boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData); if (!result) { logger.error( "Join table update es index by query sql error, destination:{}, table: {}, index: {}", @@ -737,7 +741,7 @@ public class ESSyncService { } } - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + Map paramsTmp = new LinkedHashMap<>(); for (Map.Entry> entry : tableItem.getRelationTableFields().entrySet()) { for (FieldItem fieldItem : entry.getValue()) { Object value = esTemplate @@ -747,7 +751,7 @@ public class ESSyncService { if (fieldName.equals(mapping.get_id())) { fieldName = "_id"; } - queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)); + paramsTmp.put(fieldName, value); } } @@ -758,7 +762,7 @@ public class ESSyncService { dml.getTable(), mapping.get_index()); } - boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); + boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData); if (!result) { logger.error( "Join table update es index by query whole sql error, destination:{}, table: {}, index: {}", diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index 71331873..af58595b 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -21,6 +21,12 @@ import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +/** + * ES 同步工具同类 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class ESSyncUtil { private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class); @@ -169,6 +175,24 @@ public class ESSyncUtil { location.put("lat", Double.valueOf(point[0].trim())); location.put("lon", Double.valueOf(point[1].trim())); return location; + } else if ("array".equals(esType)) { + if ("".equals(val.toString().trim())) { + res = new ArrayList<>(); + } else { + String value = val.toString(); + String separator = ","; + if (!value.contains(",")) { + if (value.contains(";")) { + separator = ";"; + } else if (value.contains("|")) { + separator = "|"; + } else if (value.contains("-")) { + separator = "-"; + } + } + String[] values = value.split(separator); + return Arrays.asList(values); + } } else if ("object".equals(esType)) { if ("".equals(val.toString().trim())) { res = new HashMap<>(); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 9ec95d54..1d53c873 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -9,7 +9,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; + +import javax.sql.DataSource; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -18,6 +19,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -31,16 +33,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; +/** + * ES 操作模板 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ public class ESTemplate { private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); - private static final int MAX_BATCH_SIZE = 10000; + private static final int MAX_BATCH_SIZE = 1000; private TransportClient transportClient; @@ -48,10 +58,6 @@ public class ESTemplate { this.transportClient = transportClient; } - public void setTransportClient(TransportClient transportClient) { - this.transportClient = transportClient; - } - /** * 插入数据 * @@ -96,18 +102,6 @@ public class ESTemplate { return commitBulkRequest(bulkRequestBuilder); } - /** - * 结合 addBulkRequest4Update 批量更新 - * - * @param consumer - * @return - */ - public boolean updateBatch(Consumer consumer) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - consumer.accept(bulkRequestBuilder); - return commitBulkRequest(bulkRequestBuilder); - } - public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, Map esFieldData) { if (mapping.get_id() != null) { @@ -120,7 +114,7 @@ public class ESTemplate { .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) .setSize(MAX_BATCH_SIZE) .get(); - for (SearchHit hit : response.getHits()) { // 理论上只有一条 + for (SearchHit hit : response.getHits()) { bulkRequestBuilder .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) .setDoc(esFieldData)); @@ -131,13 +125,63 @@ public class ESTemplate { /** * update by query * - * @param mapping - * @param queryBuilder + * @param config + * @param paramsTmp * @param esFieldData * @return */ - public boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData) { - return updateByQuery(mapping, queryBuilder, esFieldData, 1); + public boolean updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { + if (paramsTmp.isEmpty()) { + return false; + } + ESMapping mapping = config.getEsMapping(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); + + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setSize(0) + .setQuery(queryBuilder) + .get(); + long count = response.getHits().getTotalHits(); + // 如果更新量大于Max, 查询sql批量更新 + if (count > MAX_BATCH_SIZE) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + // 查询sql更新 + StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); + paramsTmp.forEach( + (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND ")); + int len = sql.length(); + sql.delete(len - 4, len); + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + int exeCount = 1; + try { + BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder; + while (rs.next()) { + Object idVal = getIdValFromRS(mapping, rs); + append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); + + if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + bulkRequestBuilderTmp = transportClient.prepareBulk(); + } + exeCount++; + } + + if (bulkRequestBuilder.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + return true; + } else { + return updateByQuery(mapping, queryBuilder, esFieldData, 1); + } } private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 2b2d59ff..8e4c6d36 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -53,8 +53,8 @@ public class HbaseAdapter implements OuterAdapter { hbaseMapping = MappingConfigLoader.load(); mappingConfigCache = new HashMap<>(); for (MappingConfig mappingConfig : hbaseMapping.values()) { - mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) - + "." + mappingConfig.getHbaseMapping().getDatabase() + "." + mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + + mappingConfig.getHbaseMapping().getDatabase() + "." + mappingConfig.getHbaseMapping().getTable(), mappingConfig); } @@ -100,33 +100,34 @@ public class HbaseAdapter implements OuterAdapter { return etlResult; } } else { - DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(task); - if (dataSource != null) { - StringBuilder resultMsg = new StringBuilder(); - boolean resSucc = true; - // ds不为空说明传入的是datasourceKey - for (MappingConfig configTmp : hbaseMapping.values()) { - // 取所有的datasourceKey为task的配置 - if (configTmp.getDataSourceKey().equals(task)) { - EtlResult etlRes = HbaseEtlService.importData(dataSource, hbaseTemplate, configTmp, params); - if (!etlRes.getSucceeded()) { - resSucc = false; - resultMsg.append(etlRes.getErrorMessage()).append("\n"); - } else { - resultMsg.append(etlRes.getResultMessage()).append("\n"); - } + StringBuilder resultMsg = new StringBuilder(); + boolean resSucc = true; + // ds不为空说明传入的是datasourceKey + for (MappingConfig configTmp : hbaseMapping.values()) { + // 取所有的destination为task的配置 + if (configTmp.getDestination().equals(task)) { + DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(configTmp.getDataSourceKey()); + if (dataSource == null) { + continue; } - } - if (resultMsg.length() > 0) { - etlResult.setSucceeded(resSucc); - if (resSucc) { - etlResult.setResultMessage(resultMsg.toString()); + EtlResult etlRes = HbaseEtlService.importData(dataSource, hbaseTemplate, configTmp, params); + if (!etlRes.getSucceeded()) { + resSucc = false; + resultMsg.append(etlRes.getErrorMessage()).append("\n"); } else { - etlResult.setErrorMessage(resultMsg.toString()); + resultMsg.append(etlRes.getResultMessage()).append("\n"); } - return etlResult; } } + if (resultMsg.length() > 0) { + etlResult.setSucceeded(resSucc); + if (resSucc) { + etlResult.setResultMessage(resultMsg.toString()); + } else { + etlResult.setErrorMessage(resultMsg.toString()); + } + return etlResult; + } } etlResult.setSucceeded(false); etlResult.setErrorMessage("Task not found"); diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index bd70c5e0..f0b4c6a2 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -10,25 +10,25 @@ spring: time-zone: GMT+8 default-property-inclusion: non_null -canal.conf: - canalServerHost: 127.0.0.1:11111 +#canal.conf: +# canalServerHost: 127.0.0.1:11111 # zookeeperHosts: slave1:2181 # bootstrapServers: slave1:6667 #or rocketmq # flatMessage: true - canalInstances: - - instance: example - groups: - - outAdapters: +# canalInstances: +# - instance: example +# groups: +# - outAdapters: # - name: logger # - name: hbase # properties: # hbase.zookeeper.quorum: 127.0.0.1 # hbase.zookeeper.property.clientPort: 2181 # zookeeper.znode.parent: /hbase - - name: es - hosts: 127.0.0.1:9300 - properties: - cluster.name: elasticsearch +# - name: es +# hosts: 127.0.0.1:9300 +# properties: +# cluster.name: elasticsearch # mqTopics: # - mqMode: kafka # topic: example @@ -44,12 +44,12 @@ canal.conf: # outAdapters: # - name: logger -adapter.conf: - datasourceConfigs: - defaultDS: - url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true - username: root - password: 121212 - adapterConfigs: - - hbase/mytest_person2.yml - - es/mytest_user.yml +#adapter.conf: +# datasourceConfigs: +# defaultDS: +# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true +# username: root +# password: 121212 +# adapterConfigs: +# - hbase/mytest_person2.yml +# - es/mytest_user.yml From beed10901d210d5fc6ad45cebeca2c0df3f27285 Mon Sep 17 00:00:00 2001 From: wuwo Date: Mon, 5 Nov 2018 16:22:15 +0800 Subject: [PATCH 39/42] destory TableMetaTSDB when stop EventParser --- .../main/resources/spring/tsdb/h2-tsdb.xml | 2 +- .../main/resources/spring/tsdb/mysql-tsdb.xml | 2 +- .../inbound/mysql/tsdb/DatabaseTableMeta.java | 22 ++++++++++++++++++- .../inbound/mysql/tsdb/MemoryTableMeta.java | 5 +++++ .../inbound/mysql/tsdb/TableMetaTSDB.java | 5 +++++ parse/src/test/resources/tsdb/h2-tsdb.xml | 2 +- parse/src/test/resources/tsdb/mysql-tsdb.xml | 2 +- 7 files changed, 35 insertions(+), 5 deletions(-) diff --git a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml index beb61236..c05b5200 100644 --- a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml +++ b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml @@ -23,7 +23,7 @@ - + diff --git a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml index 58cab638..87f407b3 100644 --- a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml +++ b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml @@ -23,7 +23,7 @@ - + diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index b0dd89ce..73a5ead8 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -72,7 +72,7 @@ public class DatabaseTableMeta implements TableMetaTSDB { @Override public Thread newThread(Runnable r) { - Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]"); + Thread thread = new Thread(r, String.format("[scheduler-table-meta-snapshot-%s]", destination)); thread.setDaemon(true); return thread; } @@ -105,6 +105,26 @@ public class DatabaseTableMeta implements TableMetaTSDB { } return true; } + + @Override + public void destory() { + if (memoryTableMeta != null) { + memoryTableMeta.destory(); + } + + if (connection != null) { + try { + connection.disconnect(); + } catch (IOException e) { + logger.error("ERROR # disconnect meta connection for address:{}", connection.getConnector() + .getAddress(), e); + } + } + + if (scheduler != null) { + scheduler.shutdown(); + } + } @Override public TableMeta find(String schema, String table) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java index 904b55a9..c53e1a08 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -58,6 +58,11 @@ public class MemoryTableMeta implements TableMetaTSDB { public boolean init(String destination) { return true; } + + @Override + public void destory() { + tableMetas.clear(); + } public boolean apply(EntryPosition position, String schema, String ddl, String extra) { tableMetas.clear(); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java index d712adbe..f968ae12 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java @@ -18,6 +18,11 @@ public interface TableMetaTSDB { */ public boolean init(String destination); + /** + * 销毁资源 + */ + public void destory(); + /** * 获取当前的表结构 */ diff --git a/parse/src/test/resources/tsdb/h2-tsdb.xml b/parse/src/test/resources/tsdb/h2-tsdb.xml index c0a3d385..dcb6bc47 100644 --- a/parse/src/test/resources/tsdb/h2-tsdb.xml +++ b/parse/src/test/resources/tsdb/h2-tsdb.xml @@ -7,7 +7,7 @@ default-autowire="byName"> - + diff --git a/parse/src/test/resources/tsdb/mysql-tsdb.xml b/parse/src/test/resources/tsdb/mysql-tsdb.xml index 099c034d..44f97203 100644 --- a/parse/src/test/resources/tsdb/mysql-tsdb.xml +++ b/parse/src/test/resources/tsdb/mysql-tsdb.xml @@ -7,7 +7,7 @@ default-autowire="byName"> - + From e608f09b53e1d0d0964fda73693320f3f66148dc Mon Sep 17 00:00:00 2001 From: wuwo Date: Mon, 5 Nov 2018 16:37:42 +0800 Subject: [PATCH 40/42] improve DatabaseTableMeta --- .../inbound/mysql/tsdb/DatabaseTableMeta.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index 73a5ead8..8896e2f5 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -48,18 +49,26 @@ public class DatabaseTableMeta implements TableMetaTSDB { private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class); private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); + private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]"); + thread.setDaemon(true); + return thread; + } + }); private String destination; private MemoryTableMeta memoryTableMeta; private MysqlConnection connection; // 查询meta信息的链接 private CanalEventFilter filter; private CanalEventFilter blackFilter; private EntryPosition lastPosition; - private ScheduledExecutorService scheduler; private MetaHistoryDAO metaHistoryDAO; private MetaSnapshotDAO metaSnapshotDAO; private int snapshotInterval = 24; private int snapshotExpire = 360; - + private ScheduledFuture scheduleSnapshotFuture; + public DatabaseTableMeta(){ } @@ -68,19 +77,10 @@ public class DatabaseTableMeta implements TableMetaTSDB { public boolean init(final String destination) { this.destination = destination; this.memoryTableMeta = new MemoryTableMeta(); - this.scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { - - @Override - public Thread newThread(Runnable r) { - Thread thread = new Thread(r, String.format("[scheduler-table-meta-snapshot-%s]", destination)); - thread.setDaemon(true); - return thread; - } - }); // 24小时生成一份snapshot if (snapshotInterval > 0) { - scheduler.scheduleWithFixedDelay(new Runnable() { + scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() { @Override public void run() { @@ -121,8 +121,8 @@ public class DatabaseTableMeta implements TableMetaTSDB { } } - if (scheduler != null) { - scheduler.shutdown(); + if (scheduleSnapshotFuture != null) { + scheduleSnapshotFuture.cancel(false); } } From 359aa1a0740d2b47d6ebdc70377b9fbdfa42c5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Mon, 5 Nov 2018 16:53:15 +0800 Subject: [PATCH 41/42] fixed compiler failed --- .../dbsync/binlog/BaseLogFetcherTest.java | 4 +- pom.xml | 57 +------------------ 2 files changed, 4 insertions(+), 57 deletions(-) diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java index d0c6746d..27ecf3db 100644 --- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java @@ -68,7 +68,7 @@ public class BaseLogFetcherTest { // update需要处理before/after System.out.println("-------> before"); parseOneRow(event, buffer, columns, false); - if (!buffer.nextOneRow(changeColumns)) { + if (!buffer.nextOneRow(changeColumns, true)) { break; } System.out.println("-------> after"); @@ -97,7 +97,7 @@ public class BaseLogFetcherTest { } ColumnInfo info = columnInfo[i]; - buffer.nextValue(info.type, info.meta); + buffer.nextValue(null , i ,info.type, info.meta); if (buffer.isNull()) { // diff --git a/pom.xml b/pom.xml index 5cb51eac..0bb4bc11 100644 --- a/pom.xml +++ b/pom.xml @@ -371,6 +371,7 @@ + + --> src/main/java src/test/java From 0d9695c3a54647a593c4b5f9476e974d36675b33 Mon Sep 17 00:00:00 2001 From: mcy Date: Mon, 5 Nov 2018 17:29:11 +0800 Subject: [PATCH 42/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9array=E5=92=8C?= =?UTF-8?q?object=E7=B1=BB=E5=9E=8B=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/es/config/ESSyncConfig.java | 37 +++++++++++++------ .../client/adapter/es/support/ESSyncUtil.java | 14 +++++++ .../client/adapter/es/support/ESTemplate.java | 30 +++++++++++---- .../src/main/resources/es/mytest_user.yml | 2 + 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java index 99d425a6..9d4a9da2 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -1,7 +1,9 @@ package com.alibaba.otter.canal.client.adapter.es.config; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; /** * ES 映射配置 @@ -58,19 +60,22 @@ public class ESSyncConfig { public static class ESMapping { - private String _index; - private String _type; - private String _id; - private String pk; - private String parent; - private String sql; - private List skips = new ArrayList<>(); - private int commitBatch = 1000; - private String etlCondition; - private boolean syncByTimestamp = false; // 是否按时间戳定时同步 - private Long syncInterval; // 同步时间间隔 + private String _index; + private String _type; + private String _id; + private String pk; + private String parent; + private String sql; + // 对象字段, 例: objFields: + // - _labels: array:; + private Map objFields = new LinkedHashMap<>(); + private List skips = new ArrayList<>(); + private int commitBatch = 1000; + private String etlCondition; + private boolean syncByTimestamp = false; // 是否按时间戳定时同步 + private Long syncInterval; // 同步时间间隔 - private SchemaItem schemaItem; // sql解析结果模型 + private SchemaItem schemaItem; // sql解析结果模型 public String get_index() { return _index; @@ -112,6 +117,14 @@ public class ESSyncConfig { this.parent = parent; } + public Map getObjFields() { + return objFields; + } + + public void setObjFields(Map objFields) { + this.objFields = objFields; + } + public List getSkips() { return skips; } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index af58595b..cf8f8fd4 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -31,6 +31,20 @@ public class ESSyncUtil { private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class); + public static Object convertToEsObj(Object val, String fieldInfo) { + if (val == null) { + return null; + } + if (fieldInfo.startsWith("array:")) { + String separator = fieldInfo.substring("array:".length()).trim(); + String[] values = val.toString().split(separator); + return Arrays.asList(values); + } else if (fieldInfo.startsWith("object")) { + return JSON.parse(val.toString()); + } + return null; + } + /** * 类型转换为Mapping中对应的类型 */ diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 1d53c873..e57b89b7 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -2,16 +2,14 @@ package com.alibaba.otter.canal.client.adapter.es.support; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; +import com.alibaba.fastjson.JSON; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -205,8 +203,14 @@ public class ESTemplate { .append(mapValue.get("lat")) .append("];"); } else { - logger.warn("Unsupported object type for script_update"); + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); } + } else if (value instanceof List) { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); } else if (value instanceof String) { sb.append("ctx._source") .append("['") @@ -325,7 +329,13 @@ public class ESTemplate { value = resultSet.getByte(columnName); } } - return ESSyncUtil.typeConvert(value, esType); + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } } public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, @@ -393,7 +403,13 @@ public class ESTemplate { value = ((Byte) value).intValue() != 0; } } - return ESSyncUtil.typeConvert(value, esType); + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } } /** diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 7b9effde..849420a3 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -10,5 +10,7 @@ esMapping: left join role b on b.id=a.role_id left join (select user_id, group_concat(label order by id desc separator ';') as labels from label group by user_id) c on c.user_id=a.id" +# objFields: +# _labels: array:; etlCondition: "where a.c_time>='{0}'" commitBatch: 3000