Merge remote-tracking branch 'upstream/master'

client-adapter/common/pom.xml

@@ -14,7 +14,7 @@
         <dependency>
             <groupId>com.alibaba.otter</groupId>
             <artifactId>canal.protocol</artifactId>
-            <version>1.1.2-SNAPSHOT</version>
+            <version>${canal_version}</version>
         </dependency>
         <dependency>
             <groupId>joda-time</groupId>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -19,6 +19,12 @@ public class CanalClientConfig {
 
     private Boolean             flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式
 
+    private Integer             batchSize;          // 批大小
+
+    private Integer             retry;              // 重试次数
+
+    private Long                timeout;            // 消费超时时间
+
     private List<MQTopic>       mqTopics;           // mq topic 列表
 
     private List<CanalInstance> canalInstances;     // tcp 模式下 canal 实例列表, 与mq模式不能共存!!
@@ -63,6 +69,30 @@ public class CanalClientConfig {
         this.flatMessage = flatMessage;
     }
 
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(Integer batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public Integer getRetry() {
+        return retry;
+    }
+
+    public void setRetry(Integer retry) {
+        this.retry = retry;
+    }
+
+    public Long getTimeout() {
+        return timeout;
+    }
+
+    public void setTimeout(Long timeout) {
+        this.timeout = timeout;
+    }
+
     public List<CanalInstance> getCanalInstances() {
         return canalInstances;
     }
@@ -73,9 +103,9 @@ public class CanalClientConfig {
 
     public static class CanalInstance {
 
-        private String             instance;      // 实例名
+        private String      instance; // 实例名
 
-        private List<Group> groups;  // 适配器分组列表
+        private List<Group> groups;   // 适配器分组列表
 
         public String getInstance() {
             return instance;
@@ -112,9 +142,9 @@ public class CanalClientConfig {
 
     public static class MQTopic {
 
-        private String      mqMode;                     // mq模式 kafka or rocketMQ
+        private String        mqMode;                     // mq模式 kafka or rocketMQ
 
-        private String      topic;                      // topic名
+        private String        topic;                      // topic名
 
         private List<MQGroup> groups = new ArrayList<>(); // 分组列表
 

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java

@@ -292,10 +292,13 @@ public class ExtensionLoader<T> {
 
     private String getJarDirectoryPath() {
         URL url = Thread.currentThread().getContextClassLoader().getResource("");
-        if (url == null) {
+        String dirtyPath;
-            throw new IllegalStateException("failed to get class loader resource");
+        if (url != null) {
+            dirtyPath = url.toString();
+        } else {
+            File file = new File("");
+            dirtyPath = file.getAbsolutePath();
         }
-        String dirtyPath = url.toString();
         String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
                                                                 // file:/ and
                                                                 // everything

client-adapter/elasticsearch/pom.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.2-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.elasticsearch</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter elasticsearch module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.19</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.fastsql</groupId>
+            <artifactId>fastsql</artifactId>
+            <version>2.0.0_preview_644</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>6.2.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>transport</artifactId>
+            <version>6.2.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java

@@ -0,0 +1,151 @@
+package com.alibaba.otter.canal.client.adapter.es;
+
+import java.net.InetAddress;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.sql.DataSource;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
+import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
+import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
+import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
+import com.alibaba.otter.canal.client.adapter.support.*;
+
+/**
+ * ES外部适配器
+ *
+ * @author rewerma 2018-10-20
+ * @version 1.0.0
+ */
+@SPI("es")
+public class ESAdapter implements OuterAdapter {
+
+    private TransportClient transportClient;
+
+    private ESSyncService   esSyncService;
+
+    public TransportClient getTransportClient() {
+        return transportClient;
+    }
+
+    public ESSyncService getEsSyncService() {
+        return esSyncService;
+    }
+
+    @Override
+    public void init(OuterAdapterConfig configuration) {
+        try {
+            ESSyncConfigLoader.load();
+
+            Map<String, String> properties = configuration.getProperties();
+            Settings.Builder settingBuilder = Settings.builder();
+            properties.forEach(settingBuilder::put);
+            Settings settings = settingBuilder.build();
+            transportClient = new PreBuiltTransportClient(settings);
+            String[] hostArray = configuration.getHosts().split(",");
+            for (String host : hostArray) {
+                int i = host.indexOf(":");
+                transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
+                    Integer.parseInt(host.substring(i + 1))));
+            }
+            ESTemplate esTemplate = new ESTemplate(transportClient);
+            esSyncService = new ESSyncService(esTemplate);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void sync(Dml dml) {
+        esSyncService.sync(dml);
+    }
+
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        ESSyncConfig config = ESSyncConfigLoader.getEsSyncConfig().get(task);
+        if (config != null) {
+            DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+            ESEtlService esEtlService = new ESEtlService(transportClient, config);
+            if (dataSource != null) {
+                return esEtlService.importData(params, false);
+            } else {
+                etlResult.setSucceeded(false);
+                etlResult.setErrorMessage("DataSource not found");
+                return etlResult;
+            }
+        } else {
+            StringBuilder resultMsg = new StringBuilder();
+            boolean resSuccess = true;
+            // ds不为空说明传入的是datasourceKey
+            for (ESSyncConfig configTmp : ESSyncConfigLoader.getEsSyncConfig().values()) {
+                // 取所有的destination为task的配置
+                if (configTmp.getDestination().equals(task)) {
+                    ESEtlService esEtlService = new ESEtlService(transportClient, configTmp);
+                    EtlResult etlRes = esEtlService.importData(params, false);
+                    if (!etlRes.getSucceeded()) {
+                        resSuccess = false;
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
+                    } else {
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
+                    }
+                }
+            }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSuccess);
+                if (resSuccess) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
+        }
+        etlResult.setSucceeded(false);
+        etlResult.setErrorMessage("Task not found");
+        return etlResult;
+    }
+
+    @Override
+    public Map<String, Object> count(String task) {
+        ESSyncConfig config = ESSyncConfigLoader.getEsSyncConfig().get(task);
+        ESMapping mapping = config.getEsMapping();
+        SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+            .setTypes(mapping.get_type())
+            .setSize(0)
+            .get();
+
+        long rowCount = response.getHits().getTotalHits();
+        Map<String, Object> res = new LinkedHashMap<>();
+        res.put("esIndex", mapping.get_index());
+        res.put("count", rowCount);
+        return res;
+    }
+
+    @Override
+    public void destroy() {
+        if (transportClient != null) {
+            transportClient.close();
+        }
+    }
+
+    @Override
+    public String getDestination(String task) {
+        ESSyncConfig config = ESSyncConfigLoader.getEsSyncConfig().get(task);
+        if (config != null) {
+            return config.getDestination();
+        }
+        return null;
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java

@@ -0,0 +1,184 @@
+package com.alibaba.otter.canal.client.adapter.es.config;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ES 映射配置
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESSyncConfig {
+
+    private String    dataSourceKey; // 数据源key
+
+    private String    destination;   // canal destination
+
+    private ESMapping esMapping;
+
+    public void validate() {
+        if (esMapping._index == null) {
+            throw new NullPointerException("esMapping._index");
+        }
+        if (esMapping._type == null) {
+            throw new NullPointerException("esMapping._type");
+        }
+        if (esMapping._id == null && esMapping.pk == null) {
+            throw new NullPointerException("esMapping._id and esMapping.pk");
+        }
+        if (esMapping.sql == null) {
+            throw new NullPointerException("esMapping.sql");
+        }
+    }
+
+    public String getDataSourceKey() {
+        return dataSourceKey;
+    }
+
+    public void setDataSourceKey(String dataSourceKey) {
+        this.dataSourceKey = dataSourceKey;
+    }
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public ESMapping getEsMapping() {
+        return esMapping;
+    }
+
+    public void setEsMapping(ESMapping esMapping) {
+        this.esMapping = esMapping;
+    }
+
+    public static class ESMapping {
+
+        private String              _index;
+        private String              _type;
+        private String              _id;
+        private String              pk;
+        private String              parent;
+        private String              sql;
+        // 对象字段, 例: objFields:
+        //              - _labels: array:;
+        private Map<String, String> objFields     = new LinkedHashMap<>();
+        private List<String>        skips           = new ArrayList<>();
+        private int                 commitBatch     = 1000;
+        private String              etlCondition;
+        private boolean             syncByTimestamp = false;                 // 是否按时间戳定时同步
+        private Long                syncInterval;                            // 同步时间间隔
+
+        private SchemaItem          schemaItem;                              // sql解析结果模型
+
+        public String get_index() {
+            return _index;
+        }
+
+        public void set_index(String _index) {
+            this._index = _index;
+        }
+
+        public String get_type() {
+            return _type;
+        }
+
+        public void set_type(String _type) {
+            this._type = _type;
+        }
+
+        public String get_id() {
+            return _id;
+        }
+
+        public void set_id(String _id) {
+            this._id = _id;
+        }
+
+        public String getPk() {
+            return pk;
+        }
+
+        public void setPk(String pk) {
+            this.pk = pk;
+        }
+
+        public String getParent() {
+            return parent;
+        }
+
+        public void setParent(String parent) {
+            this.parent = parent;
+        }
+
+        public Map<String, String> getObjFields() {
+            return objFields;
+        }
+
+        public void setObjFields(Map<String, String> objFields) {
+            this.objFields = objFields;
+        }
+
+        public List<String> getSkips() {
+            return skips;
+        }
+
+        public void setSkips(List<String> skips) {
+            this.skips = skips;
+        }
+
+        public String getSql() {
+            return sql;
+        }
+
+        public void setSql(String sql) {
+            this.sql = sql;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public Long getSyncInterval() {
+            return syncInterval;
+        }
+
+        public void setSyncInterval(Long syncInterval) {
+            this.syncInterval = syncInterval;
+        }
+
+        public boolean isSyncByTimestamp() {
+            return syncByTimestamp;
+        }
+
+        public void setSyncByTimestamp(boolean syncByTimestamp) {
+            this.syncByTimestamp = syncByTimestamp;
+        }
+
+        public SchemaItem getSchemaItem() {
+            return schemaItem;
+        }
+
+        public void setSchemaItem(SchemaItem schemaItem) {
+            this.schemaItem = schemaItem;
+        }
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java

@@ -0,0 +1,127 @@
+package com.alibaba.otter.canal.client.adapter.es.config;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+
+/**
+ * ES 配置装载器
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESSyncConfigLoader {
+
+    private static Logger                                   logger              = LoggerFactory
+        .getLogger(ESSyncConfigLoader.class);
+
+    private static final String                             BASE_PATH           = "es";
+
+    private static volatile Map<String, ESSyncConfig>       esSyncConfig        = new LinkedHashMap<>(); // 文件名对应配置
+    private static volatile Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = new LinkedHashMap<>(); // schema-table对应配置
+
+    public static Map<String, ESSyncConfig> getEsSyncConfig() {
+        return esSyncConfig;
+    }
+
+    public static Map<String, List<ESSyncConfig>> getDbTableEsSyncConfig() {
+        return dbTableEsSyncConfig;
+    }
+
+    public static synchronized void load() {
+        logger.info("## Start loading mapping config ... ");
+        Collection<String> configs = AdapterConfigs.get("es");
+        if (configs == null) {
+            return;
+        }
+        for (String c : configs) {
+            if (c == null) {
+                continue;
+            }
+            c = c.trim();
+            if (c.equals("") || c.startsWith("#")) {
+                continue;
+            }
+
+            ESSyncConfig config;
+            String configContent = null;
+
+            if (c.endsWith(".yml")) {
+                configContent = readConfigContent(BASE_PATH + "/" + c);
+            }
+
+            config = new Yaml().loadAs(configContent, ESSyncConfig.class);
+
+            try {
+                config.validate();
+                SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
+                config.getEsMapping().setSchemaItem(schemaItem);
+
+                DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+                if (dataSource == null || dataSource.getUrl() == null) {
+                    throw new RuntimeException("No data source found: " + config.getDataSourceKey());
+                }
+                Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
+                Matcher matcher = pattern.matcher(dataSource.getUrl());
+                if (!matcher.find()) {
+                    throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
+                }
+                String schema = matcher.group(2);
+
+                schemaItem.getAliasTableItems().values().forEach(tableItem -> {
+                    List<ESSyncConfig> esSyncConfigs = dbTableEsSyncConfig
+                        .computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new ArrayList<>());
+                    esSyncConfigs.add(config);
+                });
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR Config: " + c, e);
+            }
+            esSyncConfig.put(c, config);
+        }
+
+        logger.info("## Mapping config loaded");
+    }
+
+    private static String readConfigContent(String config) {
+        InputStream in = null;
+        try {
+            // 先取本地文件，再取类路径
+            File configFile = new File("config/" + config);
+            if (configFile.exists()) {
+                in = new FileInputStream(configFile);
+            } else {
+                in = ESSyncConfigLoader.class.getClassLoader().getResourceAsStream(config);
+            }
+            if (in == null) {
+                throw new RuntimeException("Config file: " + config + " not found.");
+            }
+
+            byte[] bytes = new byte[in.available()];
+            in.read(bytes);
+            return new String(bytes, StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException("Read yml config error ", e);
+        } finally {
+            try {
+                if (in != null) {
+                    in.close();
+                }
+            } catch (IOException e) {
+                // ignore
+            }
+        }
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java

@@ -0,0 +1,422 @@
+package com.alibaba.otter.canal.client.adapter.es.config;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
+
+/**
+ * ES 映射配置视图
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class SchemaItem {
+
+    private Map<String, TableItem>                aliasTableItems = new LinkedHashMap<>(); // 别名对应表名
+    private Map<String, FieldItem>                selectFields    = new LinkedHashMap<>(); // 查询字段
+    private String                                sql;
+
+    private volatile Map<String, List<TableItem>> tableItemAliases;
+    private volatile Map<String, List<FieldItem>> columnFields;
+    private volatile Boolean                      allFieldsSimple;
+
+    public void init() {
+        this.getTableItemAliases();
+        this.getColumnFields();
+        this.isAllFieldsSimple();
+        aliasTableItems.values().forEach(tableItem -> {
+            tableItem.getRelationTableFields();
+            tableItem.getRelationSelectFieldItems();
+        });
+    }
+
+    public Map<String, TableItem> getAliasTableItems() {
+        return aliasTableItems;
+    }
+
+    public void setAliasTableItems(Map<String, TableItem> aliasTableItems) {
+        this.aliasTableItems = aliasTableItems;
+    }
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    public Map<String, FieldItem> getSelectFields() {
+        return selectFields;
+    }
+
+    public void setSelectFields(Map<String, FieldItem> selectFields) {
+        this.selectFields = selectFields;
+    }
+
+    public Map<String, List<TableItem>> getTableItemAliases() {
+        if (tableItemAliases == null) {
+            synchronized (SchemaItem.class) {
+                if (tableItemAliases == null) {
+                    tableItemAliases = new LinkedHashMap<>();
+                    aliasTableItems.forEach((alias, tableItem) -> {
+                        List<TableItem> aliases = tableItemAliases
+                            .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>());
+                        aliases.add(tableItem);
+                    });
+                }
+            }
+        }
+        return tableItemAliases;
+    }
+
+    public Map<String, List<FieldItem>> getColumnFields() {
+        if (columnFields == null) {
+            synchronized (SchemaItem.class) {
+                if (columnFields == null) {
+                    columnFields = new LinkedHashMap<>();
+                    getSelectFields()
+                        .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> {
+                            TableItem tableItem = getAliasTableItems().get(columnItem.getOwner());
+                            // if (!tableItem.isSubQuery()) {
+                            List<FieldItem> fieldItems = columnFields.computeIfAbsent(
+                                columnItem.getOwner() + "." + columnItem.getColumnName(),
+                                k -> new ArrayList<>());
+                            fieldItems.add(fieldItem);
+                            // } else {
+                            // tableItem.getSubQueryFields().forEach(subQueryField -> {
+                            // List<FieldItem> fieldItems = columnFields.computeIfAbsent(
+                            // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(),
+                            // k -> new ArrayList<>());
+                            // fieldItems.add(fieldItem);
+                            // });
+                            // }
+                        }));
+                }
+            }
+        }
+        return columnFields;
+    }
+
+    public boolean isAllFieldsSimple() {
+        if (allFieldsSimple == null) {
+            synchronized (SchemaItem.class) {
+                if (allFieldsSimple == null) {
+                    allFieldsSimple = true;
+
+                    for (FieldItem fieldItem : getSelectFields().values()) {
+                        if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
+                            allFieldsSimple = false;
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+
+        return allFieldsSimple;
+    }
+
+    public TableItem getMainTable() {
+        if (!aliasTableItems.isEmpty()) {
+            return aliasTableItems.values().iterator().next();
+        } else {
+            return null;
+        }
+    }
+
+    public FieldItem getIdFieldItem(ESMapping mapping) {
+        if (mapping.get_id() != null) {
+            return getSelectFields().get(mapping.get_id());
+        } else {
+            return getSelectFields().get(mapping.getPk());
+        }
+    }
+
+    public static class TableItem {
+
+        private SchemaItem                               schemaItem;
+
+        private String                                   schema;
+        private String                                   tableName;
+        private String                                   alias;
+        private String                                   subQuerySql;
+        private List<FieldItem>                          subQueryFields = new ArrayList<>();
+        private List<RelationFieldsPair>                 relationFields = new ArrayList<>();
+
+        private boolean                                  main;
+        private boolean                                  subQuery;
+
+        private volatile Map<FieldItem, List<FieldItem>> relationTableFields;               // 当前表关联条件字段对应主表查询字段
+        private volatile List<FieldItem>                 relationSelectFieldItems;          // 子表所在主表的查询字段
+
+        public TableItem(SchemaItem schemaItem){
+            this.schemaItem = schemaItem;
+        }
+
+        public SchemaItem getSchemaItem() {
+            return schemaItem;
+        }
+
+        public void setSchemaItem(SchemaItem schemaItem) {
+            this.schemaItem = schemaItem;
+        }
+
+        public String getSchema() {
+            return schema;
+        }
+
+        public void setSchema(String schema) {
+            this.schema = schema;
+        }
+
+        public String getTableName() {
+            return tableName;
+        }
+
+        public void setTableName(String tableName) {
+            this.tableName = tableName;
+        }
+
+        public String getAlias() {
+            return alias;
+        }
+
+        public void setAlias(String alias) {
+            this.alias = alias;
+        }
+
+        public String getSubQuerySql() {
+            return subQuerySql;
+        }
+
+        public void setSubQuerySql(String subQuerySql) {
+            this.subQuerySql = subQuerySql;
+        }
+
+        public boolean isMain() {
+            return main;
+        }
+
+        public void setMain(boolean main) {
+            this.main = main;
+        }
+
+        public boolean isSubQuery() {
+            return subQuery;
+        }
+
+        public void setSubQuery(boolean subQuery) {
+            this.subQuery = subQuery;
+        }
+
+        public List<FieldItem> getSubQueryFields() {
+            return subQueryFields;
+        }
+
+        public void setSubQueryFields(List<FieldItem> subQueryFields) {
+            this.subQueryFields = subQueryFields;
+        }
+
+        public List<RelationFieldsPair> getRelationFields() {
+            return relationFields;
+        }
+
+        public void setRelationFields(List<RelationFieldsPair> relationFields) {
+            this.relationFields = relationFields;
+        }
+
+        public Map<FieldItem, List<FieldItem>> getRelationTableFields() {
+            if (relationTableFields == null) {
+                synchronized (SchemaItem.class) {
+                    if (relationTableFields == null) {
+                        relationTableFields = new LinkedHashMap<>();
+
+                        getRelationFields().forEach(relationFieldsPair -> {
+                            FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
+                            FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
+                            FieldItem currentTableRelField = null;
+                            if (getAlias().equals(leftFieldItem.getOwner())) {
+                                currentTableRelField = leftFieldItem;
+                            } else if (getAlias().equals(rightFieldItem.getOwner())) {
+                                currentTableRelField = rightFieldItem;
+                            }
+
+                            if (currentTableRelField != null) {
+                                List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
+                                    .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName());
+                                if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
+                                    relationTableFields.put(currentTableRelField, selectFieldItem);
+                                } else {
+                                    selectFieldItem = getSchemaItem().getColumnFields()
+                                        .get(rightFieldItem.getOwner() + "."
+                                             + rightFieldItem.getColumn().getColumnName());
+                                    if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
+                                        relationTableFields.put(currentTableRelField, selectFieldItem);
+                                    } else {
+                                        throw new UnsupportedOperationException(
+                                            "Relation condition column must in select columns.");
+                                    }
+                                }
+                            }
+                        });
+                    }
+                }
+            }
+            return relationTableFields;
+        }
+
+        public List<FieldItem> getRelationSelectFieldItems() {
+            if (relationSelectFieldItems == null) {
+                synchronized (SchemaItem.class) {
+                    if (relationSelectFieldItems == null) {
+                        relationSelectFieldItems = new ArrayList<>();
+                        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+                            if (fieldItem.getOwners().contains(getAlias())) {
+                                relationSelectFieldItems.add(fieldItem);
+                            }
+                        }
+                    }
+                }
+            }
+            return relationSelectFieldItems;
+        }
+    }
+
+    public static class RelationFieldsPair {
+
+        private FieldItem leftFieldItem;
+        private FieldItem rightFieldItem;
+
+        public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){
+            this.leftFieldItem = leftFieldItem;
+            this.rightFieldItem = rightFieldItem;
+        }
+
+        public FieldItem getLeftFieldItem() {
+            return leftFieldItem;
+        }
+
+        public void setLeftFieldItem(FieldItem leftFieldItem) {
+            this.leftFieldItem = leftFieldItem;
+        }
+
+        public FieldItem getRightFieldItem() {
+            return rightFieldItem;
+        }
+
+        public void setRightFieldItem(FieldItem rightFieldItem) {
+            this.rightFieldItem = rightFieldItem;
+        }
+    }
+
+    public static class FieldItem {
+
+        private String           fieldName;
+        private List<ColumnItem> columnItems = new ArrayList<>();
+        private List<String>     owners      = new ArrayList<>();
+
+        private boolean          method;
+        private boolean          binaryOp;
+
+        public String getFieldName() {
+            return fieldName;
+        }
+
+        public void setFieldName(String fieldName) {
+            this.fieldName = fieldName;
+        }
+
+        public List<ColumnItem> getColumnItems() {
+            return columnItems;
+        }
+
+        public void setColumnItems(List<ColumnItem> columnItems) {
+            this.columnItems = columnItems;
+        }
+
+        public boolean isMethod() {
+            return method;
+        }
+
+        public void setMethod(boolean method) {
+            this.method = method;
+        }
+
+        public boolean isBinaryOp() {
+            return binaryOp;
+        }
+
+        public void setBinaryOp(boolean binaryOp) {
+            this.binaryOp = binaryOp;
+        }
+
+        public List<String> getOwners() {
+            return owners;
+        }
+
+        public void setOwners(List<String> owners) {
+            this.owners = owners;
+        }
+
+        public void addColumn(ColumnItem columnItem) {
+            columnItems.add(columnItem);
+        }
+
+        public ColumnItem getColumn() {
+            if (!columnItems.isEmpty()) {
+                return columnItems.get(0);
+            } else {
+                return null;
+            }
+        }
+
+        public String getOwner() {
+            if (!owners.isEmpty()) {
+                return owners.get(0);
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            FieldItem fieldItem = (FieldItem) o;
+
+            return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null;
+        }
+
+        @Override
+        public int hashCode() {
+            return fieldName != null ? fieldName.hashCode() : 0;
+        }
+    }
+
+    public static class ColumnItem {
+
+        private String owner;
+        private String columnName;
+
+        public String getOwner() {
+            return owner;
+        }
+
+        public void setOwner(String owner) {
+            this.owner = owner;
+        }
+
+        public String getColumnName() {
+            return columnName;
+        }
+
+        public void setColumnName(String columnName) {
+            this.columnName = columnName;
+        }
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java

@@ -0,0 +1,210 @@
+package com.alibaba.otter.canal.client.adapter.es.config;
+
+import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
+import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.alibaba.fastsql.sql.SQLUtils;
+import com.alibaba.fastsql.sql.ast.SQLExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr;
+import com.alibaba.fastsql.sql.ast.statement.*;
+import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
+import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
+import com.alibaba.fastsql.sql.parser.ParserException;
+import com.alibaba.fastsql.sql.parser.SQLStatementParser;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.RelationFieldsPair;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
+
+/**
+ * ES同步指定sql格式解析
+ * 
+ * @author rewerma 2018-10-26 下午03:45:49
+ * @version 1.0.0
+ */
+public class SqlParser {
+
+    /**
+     * 解析sql
+     * 
+     * @param sql sql
+     * @return 视图对象
+     */
+    public static SchemaItem parse(String sql) {
+        try {
+            SQLStatementParser parser = new MySqlStatementParser(sql);
+            SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement();
+            MySqlSelectQueryBlock sqlSelectQueryBlock = (MySqlSelectQueryBlock) statement.getSelect().getQuery();
+
+            SchemaItem schemaItem = new SchemaItem();
+            schemaItem.setSql(SQLUtils.toMySqlString(sqlSelectQueryBlock));
+            SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom();
+            List<TableItem> tableItems = new ArrayList<>();
+            SqlParser.visitSelectTable(schemaItem, sqlTableSource, tableItems, null);
+            tableItems.forEach(tableItem -> schemaItem.getAliasTableItems().put(tableItem.getAlias(), tableItem));
+
+            List<FieldItem> fieldItems = collectSelectQueryFields(sqlSelectQueryBlock);
+            fieldItems.forEach(fieldItem -> schemaItem.getSelectFields().put(fieldItem.getFieldName(), fieldItem));
+
+            schemaItem.init();
+
+            if (schemaItem.getAliasTableItems().isEmpty() || schemaItem.getSelectFields().isEmpty()) {
+                throw new ParserException("Parse sql error");
+            }
+            return schemaItem;
+        } catch (Exception e) {
+            throw new ParserException();
+        }
+    }
+
+    /**
+     * 归集字段
+     * 
+     * @param sqlSelectQueryBlock sqlSelectQueryBlock
+     * @return 字段属性列表
+     */
+    private static List<FieldItem> collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) {
+        return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> {
+            FieldItem fieldItem = new FieldItem();
+            fieldItem.setFieldName(selectItem.getAlias());
+            visitColumn(selectItem.getExpr(), fieldItem);
+            return fieldItem;
+        }).collect(Collectors.toList());
+    }
+
+    /**
+     * 解析字段
+     * 
+     * @param expr sql expr
+     * @param fieldItem 字段属性
+     */
+    private static void visitColumn(SQLExpr expr, FieldItem fieldItem) {
+        if (expr instanceof SQLIdentifierExpr) {
+            // 无owner
+            SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr;
+            if (fieldItem.getFieldName() == null) {
+                fieldItem.setFieldName(identifierExpr.getName());
+            }
+            ColumnItem columnItem = new ColumnItem();
+            columnItem.setColumnName(identifierExpr.getName());
+            fieldItem.getOwners().add(null);
+            fieldItem.addColumn(columnItem);
+        } else if (expr instanceof SQLPropertyExpr) {
+            // 有owner
+            SQLPropertyExpr sqlPropertyExpr = (SQLPropertyExpr) expr;
+            if (fieldItem.getFieldName() == null) {
+                fieldItem.setFieldName(sqlPropertyExpr.getName());
+            }
+            fieldItem.getOwners().add(sqlPropertyExpr.getOwnernName());
+            ColumnItem columnItem = new ColumnItem();
+            columnItem.setColumnName(sqlPropertyExpr.getName());
+            columnItem.setOwner(sqlPropertyExpr.getOwnernName());
+            fieldItem.addColumn(columnItem);
+        } else if (expr instanceof SQLMethodInvokeExpr) {
+            SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr) expr;
+            fieldItem.setMethod(true);
+            for (SQLExpr sqlExpr : methodInvokeExpr.getArguments()) {
+                visitColumn(sqlExpr, fieldItem);
+            }
+        } else if (expr instanceof SQLBinaryOpExpr) {
+            SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr;
+            fieldItem.setBinaryOp(true);
+            visitColumn(sqlBinaryOpExpr.getLeft(), fieldItem);
+            visitColumn(sqlBinaryOpExpr.getRight(), fieldItem);
+        }
+    }
+
+    /**
+     * 解析表
+     * 
+     * @param schemaItem 视图对象
+     * @param sqlTableSource sqlTableSource
+     * @param tableItems 表对象列表
+     * @param tableItemTmp 表对象(临时)
+     */
+    private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource,
+                                         List<TableItem> tableItems, TableItem tableItemTmp) {
+        if (sqlTableSource instanceof SQLExprTableSource) {
+            SQLExprTableSource sqlExprTableSource = (SQLExprTableSource) sqlTableSource;
+            TableItem tableItem;
+            if (tableItemTmp != null) {
+                tableItem = tableItemTmp;
+            } else {
+                tableItem = new TableItem(schemaItem);
+            }
+            tableItem.setSchema(sqlExprTableSource.getSchema());
+            tableItem.setTableName(sqlExprTableSource.getTableName());
+            if (tableItem.getAlias() == null) {
+                tableItem.setAlias(sqlExprTableSource.getAlias());
+            }
+            if (tableItems.isEmpty()) {
+                // 第一张表为主表
+                tableItem.setMain(true);
+            }
+            tableItems.add(tableItem);
+        } else if (sqlTableSource instanceof SQLJoinTableSource) {
+            SQLJoinTableSource sqlJoinTableSource = (SQLJoinTableSource) sqlTableSource;
+            SQLTableSource leftTableSource = sqlJoinTableSource.getLeft();
+            visitSelectTable(schemaItem, leftTableSource, tableItems, null);
+            SQLTableSource rightTableSource = sqlJoinTableSource.getRight();
+            TableItem rightTableItem = new TableItem(schemaItem);
+            // 解析on条件字段
+            visitOnCondition(sqlJoinTableSource.getCondition(), rightTableItem);
+            visitSelectTable(schemaItem, rightTableSource, tableItems, rightTableItem);
+
+        } else if (sqlTableSource instanceof SQLSubqueryTableSource) {
+            SQLSubqueryTableSource subQueryTableSource = (SQLSubqueryTableSource) sqlTableSource;
+            MySqlSelectQueryBlock sqlSelectQuery = (MySqlSelectQueryBlock) subQueryTableSource.getSelect().getQuery();
+            TableItem tableItem;
+            if (tableItemTmp != null) {
+                tableItem = tableItemTmp;
+            } else {
+                tableItem = new TableItem(schemaItem);
+            }
+            tableItem.setAlias(subQueryTableSource.getAlias());
+            tableItem.setSubQuerySql(SQLUtils.toMySqlString(sqlSelectQuery));
+            tableItem.setSubQuery(true);
+            tableItem.setSubQueryFields(collectSelectQueryFields(sqlSelectQuery));
+            visitSelectTable(schemaItem, sqlSelectQuery.getFrom(), tableItems, tableItem);
+        }
+    }
+
+    /**
+     * 解析on条件
+     * 
+     * @param expr sql expr
+     * @param tableItem 表对象
+     */
+    private static void visitOnCondition(SQLExpr expr, TableItem tableItem) {
+        if (!(expr instanceof SQLBinaryOpExpr)) {
+            throw new UnsupportedOperationException();
+        }
+        SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr;
+        if (sqlBinaryOpExpr.getOperator() == BooleanAnd) {
+            visitOnCondition(sqlBinaryOpExpr.getLeft(), tableItem);
+            visitOnCondition(sqlBinaryOpExpr.getRight(), tableItem);
+        } else if (sqlBinaryOpExpr.getOperator() == Equality) {
+            FieldItem leftFieldItem = new FieldItem();
+            visitColumn(sqlBinaryOpExpr.getLeft(), leftFieldItem);
+            if (leftFieldItem.getColumnItems().size() != 1 || leftFieldItem.isMethod() || leftFieldItem.isBinaryOp()) {
+                throw new UnsupportedOperationException("Unsupported for complex of on-condition");
+            }
+            FieldItem rightFieldItem = new FieldItem();
+            visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
+            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
+                || rightFieldItem.isBinaryOp()) {
+                throw new UnsupportedOperationException("Unsupported for complex of on-condition");
+            }
+            tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));
+        } else {
+            throw new UnsupportedOperationException("Unsupported for complex of on-condition");
+        }
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java

@@ -0,0 +1,286 @@
+package com.alibaba.otter.canal.client.adapter.es.service;
+
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.sql.DataSource;
+
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil;
+import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.google.common.base.Joiner;
+
+/**
+ * ES ETL Service
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESEtlService {
+
+    private static Logger   logger = LoggerFactory.getLogger(ESEtlService.class);
+
+    private TransportClient transportClient;
+    private ESTemplate      esTemplate;
+    private ESSyncConfig    config;
+
+    public ESEtlService(TransportClient transportClient, ESSyncConfig config){
+        this.transportClient = transportClient;
+        this.esTemplate = new ESTemplate(transportClient);
+        this.config = config;
+    }
+
+    public EtlResult importData(List<String> params, boolean bulk) {
+        EtlResult etlResult = new EtlResult();
+        AtomicLong impCount = new AtomicLong();
+        List<String> errMsg = new ArrayList<>();
+        String esIndex = "";
+        if (config == null) {
+            logger.warn("esSycnCofnig is null, etl go end !");
+            etlResult.setErrorMessage("esSycnCofnig is null, etl go end !");
+            return etlResult;
+        }
+
+        ESMapping mapping = config.getEsMapping();
+
+        esIndex = mapping.get_index();
+        DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
+        Matcher matcher = pattern.matcher(dataSource.getUrl());
+        if (!matcher.find()) {
+            throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
+        }
+        String schema = matcher.group(2);
+
+        logger.info("etl from db: {},  to es index: {}", schema, esIndex);
+        long start = System.currentTimeMillis();
+        try {
+            String sql = mapping.getSql();
+
+            // 拼接条件
+            if (mapping.getEtlCondition() != null && params != null) {
+                String etlCondition = mapping.getEtlCondition();
+                int size = params.size();
+                for (int i = 0; i < size; i++) {
+                    etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
+                }
+
+                sql += " " + etlCondition;
+            }
+
+            if (logger.isDebugEnabled()) {
+                logger.debug("etl sql : {}", mapping.getSql());
+            }
+
+            if (bulk) {
+                // 获取总数
+                String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
+                long cnt = (Long) ESSyncUtil.sqlRS(dataSource, countSql, rs -> {
+                    Long count = null;
+                    try {
+                        if (rs.next()) {
+                            count = ((Number) rs.getObject(1)).longValue();
+                        }
+                    } catch (Exception e) {
+                        logger.error(e.getMessage(), e);
+                    }
+                    return count == null ? 0L : count;
+                });
+
+                // 当大于1万条记录时开启多线程
+                if (cnt >= 10000) {
+                    int threadCount = 3; // TODO 从配置读取默认为3
+                    long perThreadCnt = cnt / threadCount;
+                    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
+                    List<Future<Boolean>> futures = new ArrayList<>(threadCount);
+                    for (int i = 0; i < threadCount; i++) {
+                        long offset = i * perThreadCnt;
+                        Long size = null;
+                        if (i != threadCount - 1) {
+                            size = perThreadCnt;
+                        }
+                        String sqlFinal;
+                        if (size != null) {
+                            sqlFinal = sql + " LIMIT " + offset + "," + size;
+                        } else {
+                            sqlFinal = sql + " LIMIT " + offset + "," + cnt;
+                        }
+                        Future<Boolean> future = executor
+                            .submit(() -> executeSqlImport(dataSource, sqlFinal, mapping, impCount, errMsg));
+                        futures.add(future);
+                    }
+
+                    for (Future<Boolean> future : futures) {
+                        future.get();
+                    }
+
+                    executor.shutdown();
+                } else {
+                    executeSqlImport(dataSource, sql, mapping, impCount, errMsg);
+                }
+            } else {
+                logger.info("自动ETL，无需统计记录总条数，直接进行ETL, index: {}", esIndex);
+                executeSqlImport(dataSource, sql, mapping, impCount, errMsg);
+            }
+
+            logger.info("数据全量导入完成,一共导入 {} 条数据, 耗时: {}", impCount.get(), System.currentTimeMillis() - start);
+            etlResult.setResultMessage("导入ES索引 " + esIndex + " 数据：" + impCount.get() + " 条");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            errMsg.add(esIndex + " etl failed! ==>" + e.getMessage());
+        }
+        if (errMsg.isEmpty()) {
+            etlResult.setSucceeded(true);
+        } else {
+            etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
+        }
+        return etlResult;
+    }
+
+    private void processFailBulkResponse(BulkResponse bulkResponse, boolean hasParent) {
+        for (BulkItemResponse response : bulkResponse.getItems()) {
+            if (!response.isFailed()) {
+                continue;
+            }
+
+            if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) {
+                logger.warn(response.getFailureMessage());
+            } else {
+                logger.error("全量导入数据有误 {}", response.getFailureMessage());
+                throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage());
+            }
+        }
+    }
+
+    private boolean executeSqlImport(DataSource ds, String sql, ESMapping mapping, AtomicLong impCount,
+                                     List<String> errMsg) {
+        try {
+            ESSyncUtil.sqlRS(ds, sql, rs -> {
+                int count = 0;
+                try {
+                    BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
+
+                    long batchBegin = System.currentTimeMillis();
+                    while (rs.next()) {
+                        Map<String, Object> esFieldData = new LinkedHashMap<>();
+                        for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
+
+                            // 如果是主键字段则不插入
+                            if (fieldItem.getFieldName().equals(mapping.get_id())) {
+                                continue;
+                            }
+
+                            String fieldName = fieldItem.getFieldName();
+                            if (mapping.getSkips().contains(fieldName)) {
+                                continue;
+                            }
+
+                            Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                            esFieldData.put(fieldName, val);
+                        }
+                        Object idVal = null;
+                        if (mapping.get_id() != null) {
+                            idVal = rs.getObject(mapping.get_id());
+                        }
+
+                        if (idVal != null) {
+                            if (mapping.getParent() == null) {
+                                bulkRequestBuilder.add(transportClient
+                                    .prepareIndex(mapping.get_index(), mapping.get_type(), idVal.toString())
+                                    .setSource(esFieldData));
+                            } else {
+                                // ignore
+                            }
+                        } else {
+                            idVal = rs.getObject(mapping.getPk());
+                            if (mapping.getParent() == null) {
+                                // 删除pk对应的数据
+                                SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+                                    .setTypes(mapping.get_type())
+                                    .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
+                                    .get();
+                                for (SearchHit hit : response.getHits()) {
+                                    bulkRequestBuilder.add(transportClient
+                                        .prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId()));
+                                }
+
+                                bulkRequestBuilder
+                                    .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type())
+                                        .setSource(esFieldData));
+                            } else {
+                                // ignore
+                            }
+                        }
+
+                        if (bulkRequestBuilder.numberOfActions() % mapping.getCommitBatch() == 0
+                            && bulkRequestBuilder.numberOfActions() > 0) {
+                            long esBatchBegin = System.currentTimeMillis();
+                            BulkResponse rp = bulkRequestBuilder.execute().actionGet();
+                            if (rp.hasFailures()) {
+                                this.processFailBulkResponse(rp, Objects.nonNull(mapping.getParent()));
+                            }
+
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                                    (System.currentTimeMillis() - batchBegin),
+                                    (System.currentTimeMillis() - esBatchBegin),
+                                    bulkRequestBuilder.numberOfActions(),
+                                    mapping.get_index());
+                            }
+                            batchBegin = System.currentTimeMillis();
+                            bulkRequestBuilder = transportClient.prepareBulk();
+                        }
+                        count++;
+                        impCount.incrementAndGet();
+                    }
+
+                    if (bulkRequestBuilder.numberOfActions() > 0) {
+                        long esBatchBegin = System.currentTimeMillis();
+                        BulkResponse rp = bulkRequestBuilder.execute().actionGet();
+                        if (rp.hasFailures()) {
+                            this.processFailBulkResponse(rp, Objects.nonNull(mapping.getParent()));
+                        }
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                                (System.currentTimeMillis() - batchBegin),
+                                (System.currentTimeMillis() - esBatchBegin),
+                                bulkRequestBuilder.numberOfActions(),
+                                mapping.get_index());
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
+                    throw new RuntimeException(e);
+                }
+                return count;
+            });
+
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java

@@ -0,0 +1,863 @@
+package com.alibaba.otter.canal.client.adapter.es.service;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.sql.DataSource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
+import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil;
+import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+/**
+ * ES 同步 Service
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESSyncService {
+
+    private static Logger logger = LoggerFactory.getLogger(ESSyncService.class);
+
+    private ESTemplate    esTemplate;
+
+    public ESSyncService(ESTemplate esTemplate){
+        this.esTemplate = esTemplate;
+    }
+
+    public void sync(Dml dml) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("DML: {}", dml.toString());
+        }
+        long begin = System.currentTimeMillis();
+        String database = dml.getDatabase();
+        String table = dml.getTable();
+        List<ESSyncConfig> esSyncConfigs = ESSyncConfigLoader.getDbTableEsSyncConfig().get(database + "-" + table);
+        if (esSyncConfigs != null) {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Destination: {}, database:{}, table:{}, type:{}, effect index count: {}",
+                    dml.getDestination(),
+                    dml.getDatabase(),
+                    dml.getTable(),
+                    dml.getType(),
+                    esSyncConfigs.size());
+            }
+
+            for (ESSyncConfig config : esSyncConfigs) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace("Prepared to sync index: {}, destination: {}",
+                        config.getEsMapping().get_index(),
+                        dml.getDestination());
+                }
+                this.sync(config, dml);
+                if (logger.isTraceEnabled()) {
+                    logger.trace("Sync completed: {}, destination: {}",
+                        config.getEsMapping().get_index(),
+                        dml.getDestination());
+                }
+            }
+            if (logger.isTraceEnabled()) {
+                logger.trace("Sync elapsed time: {} ms, effect index count：{}, destination: {}",
+                    (System.currentTimeMillis() - begin),
+                    esSyncConfigs.size(),
+                    dml.getDestination());
+            }
+        }
+    }
+
+    public void sync(ESSyncConfig config, Dml dml) {
+        try {
+            // 如果是按时间戳定时更新则返回
+            if (config.getEsMapping().isSyncByTimestamp()) {
+                return;
+            }
+
+            long begin = System.currentTimeMillis();
+
+            String type = dml.getType();
+            if (type != null && type.equalsIgnoreCase("INSERT")) {
+                insert(config, dml);
+            } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
+                update(config, dml);
+            } else if (type != null && type.equalsIgnoreCase("DELETE")) {
+                delete(config, dml);
+            }
+
+            if (logger.isTraceEnabled()) {
+                logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}",
+                    (System.currentTimeMillis() - begin),
+                    dml.getDestination(),
+                    config.getEsMapping().get_index());
+            }
+        } catch (Exception e) {
+            logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 插入操作dml
+     * 
+     * @param config es配置
+     * @param dml dml数据
+     */
+    private void insert(ESSyncConfig config, Dml dml) {
+        List<Map<String, Object>> dataList = dml.getData();
+        if (dataList == null || dataList.isEmpty()) {
+            return;
+        }
+        SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
+        for (Map<String, Object> data : dataList) {
+            if (data == null || data.isEmpty()) {
+                continue;
+            }
+
+            if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
+                // ------单表 & 所有字段都为简单字段------
+                singleTableSimpleFiledInsert(config, dml, data);
+            } else {
+                // ------是主表 查询sql来插入------
+                if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
+                    mainTableInsert(config, dml, data);
+                }
+
+                // 从表的操作
+                for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
+                    if (tableItem.isMain()) {
+                        continue;
+                    }
+                    if (!tableItem.getTableName().equals(dml.getTable())) {
+                        continue;
+                    }
+                    // 关联条件出现在主表查询条件是否为简单字段
+                    boolean allFieldsSimple = true;
+                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                        if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
+                            allFieldsSimple = false;
+                            break;
+                        }
+                    }
+                    // 所有查询字段均为简单字段
+                    if (allFieldsSimple) {
+                        // 不是子查询
+                        if (!tableItem.isSubQuery()) {
+                            // ------关联表简单字段插入------
+                            Map<String, Object> esFieldData = new LinkedHashMap<>();
+                            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                                Object value = esTemplate.getValFromData(config.getEsMapping(),
+                                    data,
+                                    fieldItem.getFieldName(),
+                                    fieldItem.getColumn().getColumnName());
+                                esFieldData.put(fieldItem.getFieldName(), value);
+                            }
+
+                            joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
+                        } else {
+                            // ------关联子表简单字段插入------
+                            subTableSimpleFieldOperation(config, dml, data, null, tableItem);
+                        }
+                    } else {
+                        // ------关联子表复杂字段插入 执行全sql更新es------
+                        wholeSqlOperation(config, dml, data, null, tableItem);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * 更新操作dml
+     *
+     * @param config es配置
+     * @param dml dml数据
+     */
+    private void update(ESSyncConfig config, Dml dml) {
+        List<Map<String, Object>> dataList = dml.getData();
+        List<Map<String, Object>> oldList = dml.getOld();
+        if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) {
+            return;
+        }
+        SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
+        int i = 0;
+        for (Map<String, Object> data : dataList) {
+            Map<String, Object> old = oldList.get(i);
+            if (data == null || data.isEmpty() || old == null || old.isEmpty()) {
+                continue;
+            }
+
+            if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
+                // ------单表 & 所有字段都为简单字段------
+                singleTableSimpleFiledUpdate(config, dml, data, old);
+            } else {
+                // ------主表 查询sql来更新------
+                if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
+                    ESMapping mapping = config.getEsMapping();
+                    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+                    FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName);
+
+                    boolean idFieldSimple = true;
+                    if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) {
+                        idFieldSimple = false;
+                    }
+
+                    boolean allUpdateFieldSimple = true;
+                    out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+                        for (ColumnItem columnItem : fieldItem.getColumnItems()) {
+                            if (old.containsKey(columnItem.getColumnName())) {
+                                if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
+                                    allUpdateFieldSimple = false;
+                                    break out;
+                                }
+                            }
+                        }
+                    }
+
+                    // 不支持主键更新!!
+
+                    // 判断是否有外键更新
+                    boolean fkChanged = false;
+                    for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
+                        if (tableItem.isMain()) {
+                            continue;
+                        }
+                        boolean changed = false;
+                        for (List<FieldItem> fieldItems : tableItem.getRelationTableFields().values()) {
+                            for (FieldItem fieldItem : fieldItems) {
+                                if (old.containsKey(fieldItem.getColumn().getColumnName())) {
+                                    fkChanged = true;
+                                    changed = true;
+                                    break;
+                                }
+                            }
+                        }
+                        // 如果外键有修改,则更新所对应该表的所有查询条件数据
+                        if (changed) {
+                            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                                fieldItem.getColumnItems()
+                                    .forEach(columnItem -> old.put(columnItem.getColumnName(), null));
+                            }
+                        }
+                    }
+
+                    // 判断主键和所更新的字段是否全为简单字段
+                    if (idFieldSimple && allUpdateFieldSimple && !fkChanged) {
+                        singleTableSimpleFiledUpdate(config, dml, data, old);
+                    } else {
+                        mainTableUpdate(config, dml, data, old);
+                    }
+                }
+
+                // 从表的操作
+                for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
+                    if (tableItem.isMain()) {
+                        continue;
+                    }
+                    if (!tableItem.getTableName().equals(dml.getTable())) {
+                        continue;
+                    }
+
+                    // 关联条件出现在主表查询条件是否为简单字段
+                    boolean allFieldsSimple = true;
+                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                        if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
+                            allFieldsSimple = false;
+                            break;
+                        }
+                    }
+
+                    // 所有查询字段均为简单字段
+                    if (allFieldsSimple) {
+                        // 不是子查询
+                        if (!tableItem.isSubQuery()) {
+                            // ------关联表简单字段更新------
+                            Map<String, Object> esFieldData = new LinkedHashMap<>();
+                            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                                if (old.containsKey(fieldItem.getColumn().getColumnName())) {
+                                    Object value = esTemplate.getValFromData(config.getEsMapping(),
+                                        data,
+                                        fieldItem.getFieldName(),
+                                        fieldItem.getColumn().getColumnName());
+                                    esFieldData.put(fieldItem.getFieldName(), value);
+                                }
+                            }
+                            joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
+                        } else {
+                            // ------关联子表简单字段更新------
+                            subTableSimpleFieldOperation(config, dml, data, old, tableItem);
+                        }
+                    } else {
+                        // ------关联子表复杂字段更新 执行全sql更新es------
+                        wholeSqlOperation(config, dml, data, old, tableItem);
+                    }
+                }
+            }
+
+            i++;
+        }
+    }
+
+    /**
+     * 删除操作dml
+     *
+     * @param config es配置
+     * @param dml dml数据
+     */
+    private void delete(ESSyncConfig config, Dml dml) {
+        List<Map<String, Object>> dataList = dml.getData();
+        if (dataList == null || dataList.isEmpty()) {
+            return;
+        }
+        SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
+
+        for (Map<String, Object> data : dataList) {
+            if (data == null || data.isEmpty()) {
+                continue;
+            }
+
+            ESMapping mapping = config.getEsMapping();
+
+            // ------是主表------
+            if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
+                FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
+                // 主键为简单字段
+                if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
+                    Object idVal = esTemplate.getValFromData(mapping,
+                        data,
+                        idFieldItem.getFieldName(),
+                        idFieldItem.getColumn().getColumnName());
+
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                    boolean result = esTemplate.delete(mapping, idVal);
+                    if (!result) {
+                        logger.error("Main table delete es index error, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                } else {
+                    // ------主键带函数, 查询sql获取主键删除------
+                    mainTableDelete(config, dml, data);
+                }
+            }
+
+            // 从表的操作
+            for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
+                if (tableItem.isMain()) {
+                    continue;
+                }
+                if (!tableItem.getTableName().equals(dml.getTable())) {
+                    continue;
+                }
+
+                // 关联条件出现在主表查询条件是否为简单字段
+                boolean allFieldsSimple = true;
+                for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                    if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
+                        allFieldsSimple = false;
+                        break;
+                    }
+                }
+
+                // 所有查询字段均为简单字段
+                if (allFieldsSimple) {
+                    // 不是子查询
+                    if (!tableItem.isSubQuery()) {
+                        // ------关联表简单字段更新为null------
+                        Map<String, Object> esFieldData = new LinkedHashMap<>();
+                        for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                            esFieldData.put(fieldItem.getFieldName(), null);
+                        }
+                        joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
+                    } else {
+                        // ------关联子表简单字段更新------
+                        subTableSimpleFieldOperation(config, dml, data, null, tableItem);
+                    }
+                } else {
+                    // ------关联子表复杂字段更新 执行全sql更新es------
+                    wholeSqlOperation(config, dml, data, null, tableItem);
+                }
+            }
+        }
+    }
+
+    /**
+     * 单表简单字段insert
+     *
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行dml数据
+     */
+    private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
+        ESMapping mapping = config.getEsMapping();
+        Map<String, Object> esFieldData = new LinkedHashMap<>();
+        Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
+
+        if (logger.isTraceEnabled()) {
+            logger.trace("Single table insert ot es index, destination:{}, table: {}, index: {}, id: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                idVal);
+        }
+        boolean result = esTemplate.insert(mapping, idVal, esFieldData);
+        if (!result) {
+            logger.error("Single table insert to es index error, destination:{}, table: {}, index: {}, id: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                idVal);
+        }
+    }
+
+    /**
+     * 主表(单表)复杂字段insert
+     * 
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行dml数据
+     */
+    private void mainTableInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
+        ESMapping mapping = config.getEsMapping();
+        String sql = mapping.getSql();
+        String condition = ESSyncUtil.pkConditionSql(mapping, data);
+        sql = ESSyncUtil.appendCondition(sql, condition);
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Main table insert ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                sql.replace("\n", " "));
+        }
+        ESSyncUtil.sqlRS(ds, sql, rs -> {
+            try {
+                while (rs.next()) {
+                    Map<String, Object> esFieldData = new LinkedHashMap<>();
+                    Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData);
+
+                    if (logger.isTraceEnabled()) {
+                        logger.trace(
+                            "Main table insert ot es index by query sql, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                    boolean result = esTemplate.insert(mapping, idVal, esFieldData);
+                    if (!result) {
+                        logger.error(
+                            "Main table insert to es index by query sql error, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            return 0;
+        });
+    }
+
+    private void mainTableDelete(ESSyncConfig config, Dml dml, Map<String, Object> data) {
+        ESMapping mapping = config.getEsMapping();
+        String sql = mapping.getSql();
+        String condition = ESSyncUtil.pkConditionSql(mapping, data);
+        sql = ESSyncUtil.appendCondition(sql, condition);
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                sql.replace("\n", " "));
+        }
+        ESSyncUtil.sqlRS(ds, sql, rs -> {
+            try {
+                while (rs.next()) {
+                    Object idVal = esTemplate.getIdValFromRS(mapping, rs);
+
+                    if (logger.isTraceEnabled()) {
+                        logger.trace(
+                            "Main table delete ot es index by query sql, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                    boolean result = esTemplate.delete(mapping, idVal);
+                    if (!result) {
+                        logger.error(
+                            "Main table delete to es index by query sql error, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            return 0;
+        });
+    }
+
+    /**
+     * 关联表主表简单字段operation
+     *
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行dml数据
+     * @param tableItem 当前表配置
+     */
+    private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data,
+                                               TableItem tableItem, Map<String, Object> esFieldData) {
+        ESMapping mapping = config.getEsMapping();
+
+        Map<String, Object> paramsTmp = new LinkedHashMap<>();
+        for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
+            for (FieldItem fieldItem : entry.getValue()) {
+                if (fieldItem.getColumnItems().size() == 1) {
+                    Object value = esTemplate.getValFromData(mapping,
+                        data,
+                        fieldItem.getFieldName(),
+                        entry.getKey().getColumn().getColumnName());
+
+                    String fieldName = fieldItem.getFieldName();
+                    // 判断是否是主键
+                    if (fieldName.equals(mapping.get_id())) {
+                        fieldName = "_id";
+                    }
+                    paramsTmp.put(fieldName, value);
+                }
+            }
+        }
+
+        if (logger.isDebugEnabled()) {
+            logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index());
+        }
+        boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData);
+        if (!result) {
+            logger.error("Join table update es index by foreign key error, destination:{}, table: {}, index: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index());
+        }
+    }
+
+    /**
+     * 关联子查询, 主表简单字段operation
+     *
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行dml数据
+     * @param old 单行old数据
+     * @param tableItem 当前表配置
+     */
+    private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data,
+                                              Map<String, Object> old, TableItem tableItem) {
+        ESMapping mapping = config.getEsMapping();
+        StringBuilder sql = new StringBuilder(
+            "SELECT * FROM (" + tableItem.getSubQuerySql() + ") " + tableItem.getAlias() + " WHERE ");
+
+        for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
+            String columnName = fkFieldItem.getColumn().getColumnName();
+            Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
+            ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName);
+        }
+        int len = sql.length();
+        sql.delete(len - 5, len);
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                sql.toString().replace("\n", " "));
+        }
+        ESSyncUtil.sqlRS(ds, sql.toString(), rs -> {
+            try {
+                while (rs.next()) {
+                    Map<String, Object> esFieldData = new LinkedHashMap<>();
+
+                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                        if (old != null) {
+                            out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
+                                for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
+                                    if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
+                                        for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
+                                            if (old.containsKey(columnItem.getColumnName())) {
+                                                Object val = esTemplate.getValFromRS(mapping,
+                                                    rs,
+                                                    fieldItem.getFieldName(),
+                                                    fieldItem.getColumn().getColumnName());
+                                                esFieldData.put(fieldItem.getFieldName(), val);
+                                                break out;
+                                            }
+                                        }
+                                }
+                            }
+                        } else {
+                            Object val = esTemplate.getValFromRS(mapping,
+                                rs,
+                                fieldItem.getFieldName(),
+                                fieldItem.getColumn().getColumnName());
+                            esFieldData.put(fieldItem.getFieldName(), val);
+                        }
+                    }
+
+                    Map<String, Object> paramsTmp = new LinkedHashMap<>();
+                    for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
+                        for (FieldItem fieldItem : entry.getValue()) {
+                            if (fieldItem.getColumnItems().size() == 1) {
+                                Object value = esTemplate.getValFromRS(mapping,
+                                    rs,
+                                    fieldItem.getFieldName(),
+                                    entry.getKey().getColumn().getColumnName());
+                                String fieldName = fieldItem.getFieldName();
+                                // 判断是否是主键
+                                if (fieldName.equals(mapping.get_id())) {
+                                    fieldName = "_id";
+                                }
+                                paramsTmp.put(fieldName, value);
+                            }
+                        }
+                    }
+
+                    if (logger.isDebugEnabled()) {
+                        logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index());
+                    }
+                    boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData);
+                    if (!result) {
+                        logger.error(
+                            "Join table update es index by query sql error, destination:{}, table: {}, index: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index());
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            return 0;
+        });
+    }
+
+    /**
+     * 关联(子查询), 主表复杂字段operation, 全sql执行
+     *
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行dml数据
+     * @param tableItem 当前表配置
+     */
+    private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old,
+                                   TableItem tableItem) {
+        ESMapping mapping = config.getEsMapping();
+        StringBuilder sql = new StringBuilder(mapping.getSql() + " WHERE ");
+
+        for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
+            String columnName = fkFieldItem.getColumn().getColumnName();
+            Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
+            ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName);
+        }
+        int len = sql.length();
+        sql.delete(len - 5, len);
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                sql.toString().replace("\n", " "));
+        }
+        ESSyncUtil.sqlRS(ds, sql.toString(), rs -> {
+            try {
+                while (rs.next()) {
+                    Map<String, Object> esFieldData = new LinkedHashMap<>();
+                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
+                        if (old != null) {
+                            // 从表子查询
+                            out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
+                                for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
+                                    if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
+                                        for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
+                                            if (old.containsKey(columnItem.getColumnName())) {
+                                                Object val = esTemplate.getValFromRS(mapping,
+                                                    rs,
+                                                    fieldItem.getFieldName(),
+                                                    fieldItem.getFieldName());
+                                                esFieldData.put(fieldItem.getFieldName(), val);
+                                                break out;
+                                            }
+                                        }
+                                }
+                            }
+                            // 从表非子查询
+                            for (FieldItem fieldItem1 : tableItem.getRelationSelectFieldItems()) {
+                                if (fieldItem1.equals(fieldItem)) {
+                                    for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
+                                        if (old.containsKey(columnItem.getColumnName())) {
+                                            Object val = esTemplate.getValFromRS(mapping,
+                                                rs,
+                                                fieldItem.getFieldName(),
+                                                fieldItem.getFieldName());
+                                            esFieldData.put(fieldItem.getFieldName(), val);
+                                            break;
+                                        }
+                                    }
+                                }
+                            }
+                        } else {
+                            Object val = esTemplate
+                                .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
+                            esFieldData.put(fieldItem.getFieldName(), val);
+                        }
+                    }
+
+                    Map<String, Object> paramsTmp = new LinkedHashMap<>();
+                    for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
+                        for (FieldItem fieldItem : entry.getValue()) {
+                            Object value = esTemplate
+                                .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
+                            String fieldName = fieldItem.getFieldName();
+                            // 判断是否是主键
+                            if (fieldName.equals(mapping.get_id())) {
+                                fieldName = "_id";
+                            }
+                            paramsTmp.put(fieldName, value);
+                        }
+                    }
+
+                    if (logger.isDebugEnabled()) {
+                        logger.trace(
+                            "Join table update es index by query whole sql, destination:{}, table: {}, index: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index());
+                    }
+                    boolean result = esTemplate.updateByQuery(config, paramsTmp, esFieldData);
+                    if (!result) {
+                        logger.error(
+                            "Join table update es index by query whole sql error, destination:{}, table: {}, index: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index());
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            return 0;
+        });
+    }
+
+    /**
+     * 单表简单字段update
+     *
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行data数据
+     * @param old 单行old数据
+     */
+    private void singleTableSimpleFiledUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data,
+                                              Map<String, Object> old) {
+        ESMapping mapping = config.getEsMapping();
+        Map<String, Object> esFieldData = new LinkedHashMap<>();
+
+        Object idVal = esTemplate.getESDataFromDmlData(mapping, data, old, esFieldData);
+
+        if (logger.isTraceEnabled()) {
+            logger.trace("Main table update ot es index, destination:{}, table: {}, index: {}, id: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                idVal);
+        }
+        boolean result = esTemplate.update(mapping, idVal, esFieldData);
+        if (!result) {
+            logger.error("Main table update to es index error, destination:{}, table: {}, index: {}, id: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                idVal);
+        }
+    }
+
+    /**
+     * 主表(单表)复杂字段update
+     *
+     * @param config es配置
+     * @param dml dml信息
+     * @param data 单行dml数据
+     */
+    private void mainTableUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old) {
+        ESMapping mapping = config.getEsMapping();
+        String sql = mapping.getSql();
+        String condition = ESSyncUtil.pkConditionSql(mapping, data);
+        sql = ESSyncUtil.appendCondition(sql, condition);
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Main table update ot es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
+                config.getDestination(),
+                dml.getTable(),
+                mapping.get_index(),
+                sql.replace("\n", " "));
+        }
+        ESSyncUtil.sqlRS(ds, sql, rs -> {
+            try {
+                while (rs.next()) {
+                    Map<String, Object> esFieldData = new LinkedHashMap<>();
+                    Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData);
+
+                    if (logger.isTraceEnabled()) {
+                        logger.trace(
+                            "Main table update ot es index by query sql, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                    boolean result = esTemplate.update(mapping, idVal, esFieldData);
+                    if (!result) {
+                        logger.error(
+                            "Main table update to es index by query sql error, destination:{}, table: {}, index: {}, id: {}",
+                            config.getDestination(),
+                            dml.getTable(),
+                            mapping.get_index(),
+                            idVal);
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            return 0;
+        });
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java

@@ -0,0 +1,335 @@
+package com.alibaba.otter.canal.client.adapter.es.support;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.util.*;
+import java.util.Date;
+import java.util.function.Function;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.codec.binary.Base64;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
+
+/**
+ * ES 同步工具同类
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESSyncUtil {
+
+    private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class);
+
+    public static Object convertToEsObj(Object val, String fieldInfo) {
+        if (val == null) {
+            return null;
+        }
+        if (fieldInfo.startsWith("array:")) {
+            String separator = fieldInfo.substring("array:".length()).trim();
+            String[] values = val.toString().split(separator);
+            return Arrays.asList(values);
+        } else if (fieldInfo.startsWith("object")) {
+            return JSON.parse(val.toString());
+        }
+        return null;
+    }
+
+    /**
+     * 类型转换为Mapping中对应的类型
+     */
+    public static Object typeConvert(Object val, String esType) {
+        if (val == null) {
+            return null;
+        }
+        if (esType == null) {
+            return val;
+        }
+        Object res = null;
+        if ("integer".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).intValue();
+            } else {
+                res = Integer.parseInt(val.toString());
+            }
+        } else if ("long".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).longValue();
+            } else {
+                res = Long.parseLong(val.toString());
+            }
+        } else if ("short".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).shortValue();
+            } else {
+                res = Short.parseShort(val.toString());
+            }
+        } else if ("byte".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).byteValue();
+            } else {
+                res = Byte.parseByte(val.toString());
+            }
+        } else if ("double".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).doubleValue();
+            } else {
+                res = Double.parseDouble(val.toString());
+            }
+        } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).floatValue();
+            } else {
+                res = Float.parseFloat(val.toString());
+            }
+        } else if ("boolean".equals(esType)) {
+            if (val instanceof Boolean) {
+                res = val;
+            } else if (val instanceof Number) {
+                int v = ((Number) val).intValue();
+                res = v != 0;
+            } else {
+                res = Boolean.parseBoolean(val.toString());
+            }
+        } else if ("date".equals(esType)) {
+            if (val instanceof java.sql.Time) {
+                DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
+                if (dateTime.getMillisOfSecond() != 0) {
+                    res = dateTime.toString("HH:mm:ss.SSS");
+                } else {
+                    res = dateTime.toString("HH:mm:ss");
+                }
+            } else if (val instanceof java.sql.Timestamp) {
+                DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
+                if (dateTime.getMillisOfSecond() != 0) {
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00");
+                } else {
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00");
+                }
+            } else if (val instanceof java.sql.Date || val instanceof Date) {
+                DateTime dateTime;
+                if (val instanceof java.sql.Date) {
+                    dateTime = new DateTime(((java.sql.Date) val).getTime());
+                } else {
+                    dateTime = new DateTime(((Date) val).getTime());
+                }
+                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
+                    && dateTime.getMillisOfSecond() == 0) {
+                    res = dateTime.toString("yyyy-MM-dd");
+                } else {
+                    if (dateTime.getMillisOfSecond() != 0) {
+                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00");
+                    } else {
+                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00");
+                    }
+                }
+            } else if (val instanceof Long) {
+                DateTime dateTime = new DateTime(((Long) val).longValue());
+                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
+                    && dateTime.getMillisOfSecond() == 0) {
+                    res = dateTime.toString("yyyy-MM-dd");
+                } else if (dateTime.getMillisOfSecond() != 0) {
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00");
+                } else {
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00");
+                }
+            } else if (val instanceof String) {
+                String v = ((String) val).trim();
+                if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
+                    && v.charAt(13) == ':' && v.charAt(16) == ':') {
+                    String dt = v.substring(0, 10) + "T" + v.substring(11);
+                    DateTime dateTime = new DateTime(dt);
+                    if (dateTime.getMillisOfSecond() != 0) {
+                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00");
+                    } else {
+                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00");
+                    }
+                } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
+                    DateTime dateTime = new DateTime(v);
+                    res = dateTime.toString("yyyy-MM-dd");
+                }
+            }
+        } else if ("binary".equals(esType)) {
+            if (val instanceof byte[]) {
+                Base64 base64 = new Base64();
+                res = base64.encodeAsString((byte[]) val);
+            } else if (val instanceof Blob) {
+                byte[] b = blobToBytes((Blob) val);
+                Base64 base64 = new Base64();
+                res = base64.encodeAsString(b);
+            } else if (val instanceof String) {
+                // 对应canal中的单字节编码
+                byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
+                Base64 base64 = new Base64();
+                res = base64.encodeAsString(b);
+            }
+        } else if ("geo_point".equals(esType)) {
+            if (!(val instanceof String)) {
+                logger.error("es type is geo_point, but source type is not String");
+                return val;
+            }
+
+            if (!((String) val).contains(",")) {
+                logger.error("es type is geo_point, source value not contains ',' separator");
+                return val;
+            }
+
+            String[] point = ((String) val).split(",");
+            Map<String, Double> location = new HashMap<>();
+            location.put("lat", Double.valueOf(point[0].trim()));
+            location.put("lon", Double.valueOf(point[1].trim()));
+            return location;
+        } else if ("array".equals(esType)) {
+            if ("".equals(val.toString().trim())) {
+                res = new ArrayList<>();
+            } else {
+                String value = val.toString();
+                String separator = ",";
+                if (!value.contains(",")) {
+                    if (value.contains(";")) {
+                        separator = ";";
+                    } else if (value.contains("|")) {
+                        separator = "|";
+                    } else if (value.contains("-")) {
+                        separator = "-";
+                    }
+                }
+                String[] values = value.split(separator);
+                return Arrays.asList(values);
+            }
+        } else if ("object".equals(esType)) {
+            if ("".equals(val.toString().trim())) {
+                res = new HashMap<>();
+            } else {
+                res = JSON.parseObject(val.toString(), Map.class);
+            }
+        } else {
+            // 其他类全以字符串处理
+            res = val.toString();
+        }
+
+        return res;
+    }
+
+    /**
+     * Blob转byte[]
+     */
+    private static byte[] blobToBytes(Blob blob) {
+        try (InputStream is = blob.getBinaryStream()) {
+            byte[] b = new byte[(int) blob.length()];
+            is.read(b);
+            return b;
+        } catch (IOException | SQLException e) {
+            logger.error(e.getMessage());
+            return null;
+        }
+    }
+
+    /**
+     * 拼接主键条件
+     * 
+     * @param mapping
+     * @param data
+     * @return
+     */
+    public static String pkConditionSql(ESMapping mapping, Map<String, Object> data) {
+        Set<ColumnItem> idColumns = new LinkedHashSet<>();
+        SchemaItem schemaItem = mapping.getSchemaItem();
+
+        TableItem mainTable = schemaItem.getMainTable();
+
+        for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) {
+            if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null)
+                || (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) {
+                idColumns.add(idColumnItem);
+            }
+        }
+
+        if (idColumns.isEmpty()) {
+            throw new RuntimeException("Not found primary key field in main table");
+        }
+
+        // 拼接condition
+        StringBuilder condition = new StringBuilder(" ");
+        for (ColumnItem idColumn : idColumns) {
+            Object idVal = data.get(idColumn.getColumnName());
+            if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append(".");
+            condition.append(idColumn.getColumnName()).append("=");
+            if (idVal instanceof String) {
+                condition.append("'").append(idVal).append("' AND ");
+            } else {
+                condition.append(idVal).append(" AND ");
+            }
+        }
+
+        if (condition.toString().endsWith("AND ")) {
+            int len2 = condition.length();
+            condition.delete(len2 - 4, len2);
+        }
+        return condition.toString();
+    }
+
+    public static String appendCondition(String sql, String condition) {
+        return sql + " WHERE " + condition + " ";
+    }
+
+    public static void appendCondition(StringBuilder sql, Object value, String owner, String columnName) {
+        if (value instanceof String) {
+            sql.append(owner).append(".").append(columnName).append("='").append(value).append("'  AND ");
+        } else {
+            sql.append(owner).append(".").append(columnName).append("=").append(value).append("  AND ");
+        }
+    }
+
+    /**
+     * 执行查询sql
+     */
+    public static Object sqlRS(DataSource ds, String sql, Function<ResultSet, Object> fun) {
+        Connection conn = null;
+        Statement smt = null;
+        ResultSet rs = null;
+        try {
+            conn = ds.getConnection();
+            smt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+            smt.setFetchSize(Integer.MIN_VALUE);
+            rs = smt.executeQuery(sql);
+
+            return fun.apply(rs);
+        } catch (SQLException e) {
+            logger.error("sqlRs has error, sql: {} ", sql);
+            throw new RuntimeException(e);
+        } finally {
+            if (rs != null) {
+                try {
+                    rs.close();
+                } catch (SQLException e) {
+                    logger.error("error to close result set");
+                }
+            }
+            if (smt != null) {
+                try {
+                    smt.close();
+                } catch (SQLException e) {
+                    logger.error("error to close statement");
+                }
+            }
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    logger.error("error to close db connection");
+                }
+            }
+        }
+    }
+}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java

@@ -0,0 +1,526 @@
+package com.alibaba.otter.canal.client.adapter.es.support;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import javax.sql.DataSource;
+
+import com.alibaba.fastjson.JSON;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.UpdateByQueryAction;
+import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.SearchHit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+
+/**
+ * ES 操作模板
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESTemplate {
+
+    private static final Logger logger         = LoggerFactory.getLogger(ESTemplate.class);
+
+    private static final int    MAX_BATCH_SIZE = 1000;
+
+    private TransportClient     transportClient;
+
+    public ESTemplate(TransportClient transportClient){
+        this.transportClient = transportClient;
+    }
+
+    /**
+     * 插入数据
+     * 
+     * @param mapping
+     * @param pkVal
+     * @param esFieldData
+     * @return
+     */
+    public boolean insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
+        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
+        if (mapping.get_id() != null) {
+            bulkRequestBuilder
+                .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString())
+                    .setSource(esFieldData));
+        } else {
+            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+                .setTypes(mapping.get_type())
+                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+                .setSize(MAX_BATCH_SIZE)
+                .get();
+            for (SearchHit hit : response.getHits()) {
+                bulkRequestBuilder
+                    .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId()));
+            }
+            bulkRequestBuilder
+                .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData));
+        }
+        return commitBulkRequest(bulkRequestBuilder);
+    }
+
+    /**
+     * 根据主键更新数据
+     * 
+     * @param mapping
+     * @param pkVal
+     * @param esFieldData
+     * @return
+     */
+    public boolean update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
+        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
+        append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData);
+        return commitBulkRequest(bulkRequestBuilder);
+    }
+
+    public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal,
+                              Map<String, Object> esFieldData) {
+        if (mapping.get_id() != null) {
+            bulkRequestBuilder
+                .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString())
+                    .setDoc(esFieldData));
+        } else {
+            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+                .setTypes(mapping.get_type())
+                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+                .setSize(MAX_BATCH_SIZE)
+                .get();
+            for (SearchHit hit : response.getHits()) {
+                bulkRequestBuilder
+                    .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId())
+                        .setDoc(esFieldData));
+            }
+        }
+    }
+
+    /**
+     * update by query
+     *
+     * @param config
+     * @param paramsTmp
+     * @param esFieldData
+     * @return
+     */
+    public boolean updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
+        if (paramsTmp.isEmpty()) {
+            return false;
+        }
+        ESMapping mapping = config.getEsMapping();
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
+
+        SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+            .setTypes(mapping.get_type())
+            .setSize(0)
+            .setQuery(queryBuilder)
+            .get();
+        long count = response.getHits().getTotalHits();
+        // 如果更新量大于Max, 查询sql批量更新
+        if (count > MAX_BATCH_SIZE) {
+            BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
+
+            DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+            // 查询sql更新
+            StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
+            paramsTmp.forEach(
+                (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND "));
+            int len = sql.length();
+            sql.delete(len - 4, len);
+            ESSyncUtil.sqlRS(ds, sql.toString(), rs -> {
+                int exeCount = 1;
+                try {
+                    BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder;
+                    while (rs.next()) {
+                        Object idVal = getIdValFromRS(mapping, rs);
+                        append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData);
+
+                        if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) {
+                            commitBulkRequest(bulkRequestBuilderTmp);
+                            bulkRequestBuilderTmp = transportClient.prepareBulk();
+                        }
+                        exeCount++;
+                    }
+
+                    if (bulkRequestBuilder.numberOfActions() > 0) {
+                        commitBulkRequest(bulkRequestBuilderTmp);
+                    }
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+                return 0;
+            });
+            return true;
+        } else {
+            return updateByQuery(mapping, queryBuilder, esFieldData, 1);
+        }
+    }
+
+    private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map<String, Object> esFieldData,
+                                  int counter) {
+        if (CollectionUtils.isEmpty(esFieldData)) {
+            return true;
+        }
+
+        StringBuilder sb = new StringBuilder();
+        esFieldData.forEach((key, value) -> {
+            if (value instanceof Map) {
+                HashMap mapValue = (HashMap) value;
+                if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) {
+                    sb.append("ctx._source")
+                        .append("['")
+                        .append(key)
+                        .append("']")
+                        .append(" = [")
+                        .append(mapValue.get("lon"))
+                        .append(", ")
+                        .append(mapValue.get("lat"))
+                        .append("];");
+                } else {
+                    sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = ");
+                    sb.append(JSON.toJSONString(value));
+                    sb.append(";");
+                }
+            } else if (value instanceof List) {
+                sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = ");
+                sb.append(JSON.toJSONString(value));
+                sb.append(";");
+            } else if (value instanceof String) {
+                sb.append("ctx._source")
+                    .append("['")
+                    .append(key)
+                    .append("']")
+                    .append(" = '")
+                    .append(value)
+                    .append("';");
+            } else {
+                sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";");
+            }
+        });
+        String scriptLine = sb.toString();
+        if (logger.isTraceEnabled()) {
+            logger.trace(scriptLine);
+        }
+
+        UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient);
+        updateByQuery.source(mapping.get_index())
+            .abortOnVersionConflict(false)
+            .filter(queryBuilder)
+            .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap()));
+
+        BulkByScrollResponse response = updateByQuery.get();
+        if (logger.isTraceEnabled()) {
+            logger.trace("updateByQuery response: {}", response.getStatus());
+        }
+        if (!CollectionUtils.isEmpty(response.getSearchFailures())) {
+            logger.error("script update_for_search has search error: " + response.getBulkFailures());
+            return false;
+        }
+
+        if (!CollectionUtils.isEmpty(response.getBulkFailures())) {
+            logger.error("script update_for_search has update error: " + response.getBulkFailures());
+            return false;
+        }
+
+        if (response.getStatus().getVersionConflicts() > 0) {
+            if (counter >= 3) {
+                logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter);
+                return false;
+            }
+            logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行...");
+            try {
+                TimeUnit.SECONDS.sleep(1);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+            return updateByQuery(mapping, queryBuilder, esFieldData, ++counter);
+        }
+
+        return true;
+    }
+
+    /**
+     * 通过主键删除数据
+     *
+     * @param mapping
+     * @param pkVal
+     * @return
+     */
+    public boolean delete(ESMapping mapping, Object pkVal) {
+        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
+        if (mapping.get_id() != null) {
+            bulkRequestBuilder
+                .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString()));
+        } else {
+            SearchResponse response = transportClient.prepareSearch(mapping.get_index())
+                .setTypes(mapping.get_type())
+                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+                .setSize(MAX_BATCH_SIZE)
+                .get();
+            for (SearchHit hit : response.getHits()) {
+                bulkRequestBuilder
+                    .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId()));
+            }
+        }
+        return commitBulkRequest(bulkRequestBuilder);
+    }
+
+    /**
+     * 批量提交
+     *
+     * @param bulkRequestBuilder
+     * @return
+     */
+    private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) {
+        if (bulkRequestBuilder.numberOfActions() > 0) {
+            BulkResponse response = bulkRequestBuilder.execute().actionGet();
+            if (response.hasFailures()) {
+                for (BulkItemResponse itemResponse : response.getItems()) {
+                    if (!itemResponse.isFailed()) {
+                        continue;
+                    }
+
+                    if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
+                        logger.warn(itemResponse.getFailureMessage());
+                    } else {
+                        logger.error("ES sync commit error: {}", itemResponse.getFailureMessage());
+                    }
+                }
+            }
+
+            return !response.hasFailures();
+        }
+        return true;
+    }
+
+    public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName,
+                               String columnName) throws SQLException {
+        String esType = getEsType(mapping, fieldName);
+
+        Object value = resultSet.getObject(columnName);
+        if (value instanceof Boolean) {
+            if (!"boolean".equals(esType)) {
+                value = resultSet.getByte(columnName);
+            }
+        }
+
+        // 如果是对象类型
+        if (mapping.getObjFields().containsKey(fieldName)) {
+            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
+        } else {
+            return ESSyncUtil.typeConvert(value, esType);
+        }
+    }
+
+    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,
+                                  Map<String, Object> esFieldData) throws SQLException {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = value;
+            }
+
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
+                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                esFieldData.put(fieldItem.getFieldName(), value);
+            }
+        }
+        return resultIdVal;
+    }
+
+    public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = value;
+                break;
+            }
+        }
+        return resultIdVal;
+    }
+
+    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
+                                  Map<String, Object> esFieldData) throws SQLException {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
+            }
+
+            for (ColumnItem columnItem : fieldItem.getColumnItems()) {
+                if (dmlOld.containsKey(columnItem.getColumnName())
+                    && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                    esFieldData.put(fieldItem.getFieldName(),
+                        getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
+                    break;
+                }
+            }
+        }
+        return resultIdVal;
+    }
+
+    public Object getValFromData(ESMapping mapping, Map<String, Object> dmlData, String fieldName, String columnName) {
+        String esType = getEsType(mapping, fieldName);
+        Object value = dmlData.get(columnName);
+        if (value instanceof Byte) {
+            if ("boolean".equals(esType)) {
+                value = ((Byte) value).intValue() != 0;
+            }
+        }
+
+        // 如果是对象类型
+        if (mapping.getObjFields().containsKey(fieldName)) {
+            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
+        } else {
+            return ESSyncUtil.typeConvert(value, esType);
+        }
+    }
+
+    /**
+     * 将dml的data转换为es的data
+     *
+     * @param mapping 配置mapping
+     * @param dmlData dml data
+     * @param esFieldData es data
+     * @return 返回 id 值
+     */
+    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
+                                       Map<String, Object> esFieldData) {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
+            Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = value;
+            }
+
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
+                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                esFieldData.put(fieldItem.getFieldName(), value);
+            }
+        }
+        return resultIdVal;
+    }
+
+    /**
+     * 将dml的data, old转换为es的data
+     *
+     * @param mapping 配置mapping
+     * @param dmlData dml data
+     * @param esFieldData es data
+     * @return 返回 id 值
+     */
+    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> dmlOld,
+                                       Map<String, Object> esFieldData) {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
+            }
+
+            if (dmlOld.get(columnName) != null && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                esFieldData.put(fieldItem.getFieldName(),
+                    getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
+            }
+        }
+        return resultIdVal;
+    }
+
+    /**
+     * es 字段类型本地缓存
+     */
+    private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>();
+
+    /**
+     * 获取es mapping中的属性类型
+     *
+     * @param mapping mapping配置
+     * @param fieldName 属性名
+     * @return 类型
+     */
+    @SuppressWarnings("unchecked")
+    private String getEsType(ESMapping mapping, String fieldName) {
+        String key = mapping.get_index() + "-" + mapping.get_type();
+        Map<String, String> fieldType = esFieldTypes.get(key);
+        if (fieldType == null) {
+            ImmutableOpenMap<String, MappingMetaData> mappings;
+            try {
+                mappings = transportClient.admin()
+                    .cluster()
+                    .prepareState()
+                    .execute()
+                    .actionGet()
+                    .getState()
+                    .getMetaData()
+                    .getIndices()
+                    .get(mapping.get_index())
+                    .getMappings();
+            } catch (NullPointerException e) {
+                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
+            }
+            MappingMetaData mappingMetaData = mappings.get(mapping.get_type());
+            if (mappingMetaData == null) {
+                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
+            }
+
+            fieldType = new LinkedHashMap<>();
+
+            Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
+            Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
+            for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
+                Map<String, Object> value = (Map<String, Object>) entry.getValue();
+                if (value.containsKey("properties")) {
+                    fieldType.put(entry.getKey(), "object");
+                } else {
+                    fieldType.put(entry.getKey(), (String) value.get("type"));
+                }
+            }
+            esFieldTypes.put(key, fieldType);
+        }
+
+        return fieldType.get(fieldName);
+    }
+}

client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+es=com.alibaba.otter.canal.client.adapter.es.ESAdapter

client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml

@@ -0,0 +1,16 @@
+dataSourceKey: defaultDS
+destination: example
+esMapping:
+  _index: mytest_user
+  _type: _doc
+  _id: _id
+#  pk: id
+  sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,
+        a.c_time as _c_time, c.labels as _labels from user a
+        left join role b on b.id=a.role_id
+        left join (select user_id, group_concat(label order by id desc separator ';') as labels from label
+        group by user_id) c on c.user_id=a.id"
+#  objFields:
+#    _labels: array:;
+  etlCondition: "where a.c_time>='{0}'"
+  commitBatch: 3000

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java

@@ -0,0 +1,40 @@
+package com.alibaba.otter.canal.client.adapter.es.test;
+
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+
+public class ConfigLoadTest {
+
+    @Before
+    public void before() {
+        AdapterConfigs.put("es", "mytest_user.yml");
+        // 加载数据源连接池
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+    }
+
+    @Test
+    public void testLoad() {
+        ESSyncConfigLoader.load();
+        Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.getEsSyncConfig();
+        ESSyncConfig config = configMap.get("mytest_user.yml");
+        Assert.assertNotNull(config);
+        Assert.assertEquals("defaultDS", config.getDataSourceKey());
+        ESSyncConfig.ESMapping esMapping = config.getEsMapping();
+        Assert.assertEquals("mytest_user", esMapping.get_index());
+        Assert.assertEquals("_doc", esMapping.get_type());
+        Assert.assertEquals("id", esMapping.get_id());
+        Assert.assertNotNull(esMapping.getSql());
+
+        Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = ESSyncConfigLoader.getDbTableEsSyncConfig();
+        Assert.assertFalse(dbTableEsSyncConfig.isEmpty());
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java

@@ -0,0 +1,47 @@
+package com.alibaba.otter.canal.client.adapter.es.test;
+
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
+import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
+
+public class SqlParseTest {
+
+    @Test
+    public void parseTest() {
+        String sql = "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a "
+                     + "left join role b on a.role_id=b.id "
+                     + "left join (select user_id, group_concat(label,',') as labels from user_label "
+                     + "group by user_id) c on c.user_id=a.id";
+        SchemaItem schemaItem = SqlParser.parse(sql);
+
+        // 通过表名找 TableItem
+        List<TableItem> tableItems = schemaItem.getTableItemAliases().get("user_label".toLowerCase());
+        tableItems.forEach(tableItem -> Assert.assertEquals("c", tableItem.getAlias()));
+
+        TableItem tableItem = tableItems.get(0);
+        Assert.assertFalse(tableItem.isMain());
+        Assert.assertTrue(tableItem.isSubQuery());
+        // 通过字段名找 FieldItem
+        List<FieldItem> fieldItems = schemaItem.getColumnFields().get(tableItem.getAlias() + ".label".toLowerCase());
+        fieldItems.forEach(
+            fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName()));
+
+        // 获取当前表关联条件字段
+        Map<FieldItem, List<FieldItem>> relationTableFields = tableItem.getRelationTableFields();
+        relationTableFields.keySet()
+            .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName()));
+
+        // 获取关联字段在select中的对应字段
+        // List<FieldItem> relationSelectFieldItem =
+        // tableItem.getRelationKeyFieldItems();
+        // relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels",
+        // fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName()));
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java

@@ -0,0 +1,40 @@
+package com.alibaba.otter.canal.client.adapter.es.test;
+
+import java.sql.SQLException;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+public class TestConstant {
+
+    public final static String    jdbcUrl      = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
+    public final static String    jdbcUser     = "root";
+    public final static String    jdbcPassword = "121212";
+
+    public final static String    esHosts      = "127.0.0.1:9300";
+    public final static String    clusterNmae  = "elasticsearch";
+
+    public static DruidDataSource dataSource;
+
+    static {
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUser);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setPoolPreparedStatements(false);
+        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
+        dataSource.setValidationQuery("select 1");
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java

@@ -0,0 +1,68 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.sql.DataSource;
+
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.es.test.TestConstant;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+
+public class Common {
+
+    public static ESAdapter init() {
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+
+        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
+        outerAdapterConfig.setName("es");
+        outerAdapterConfig.setHosts(TestConstant.esHosts);
+        Map<String, String> properties = new HashMap<>();
+        properties.put("cluster.name", TestConstant.clusterNmae);
+        outerAdapterConfig.setProperties(properties);
+
+        ESAdapter esAdapter = new ESAdapter();
+        esAdapter.init(outerAdapterConfig);
+        return esAdapter;
+    }
+
+    public static void sqlExe(DataSource dataSource, String sql) {
+        Connection conn = null;
+        Statement stmt = null;
+        try {
+            conn = dataSource.getConnection();
+            conn.setAutoCommit(false);
+            stmt = conn.createStatement();
+            stmt.execute(sql);
+            conn.commit();
+        } catch (Exception e) {
+            if (conn != null) {
+                try {
+                    conn.rollback();
+                } catch (SQLException e1) {
+                    // ignore
+                }
+            }
+            e.printStackTrace();
+        } finally {
+            if (stmt != null) {
+                try {
+                    stmt.close();
+                } catch (SQLException e) {
+                    // ignore
+                }
+            }
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    // ignore
+                }
+            }
+        }
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java

@@ -0,0 +1,122 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.util.*;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+import javax.sql.DataSource;
+
+public class LabelSyncJoinSub2Test {
+
+    private ESAdapter esAdapter;
+
+    @Before
+    public void init() {
+        AdapterConfigs.put("es", "mytest_user_join_sub2.yml");
+        esAdapter = Common.init();
+    }
+
+    /**
+     * 带函数子查询从表插入
+     */
+    @Test
+    public void test01() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"delete from label where id=1 or id=2");
+        Common.sqlExe(ds,"insert into label (id,user_id,label) values (1,1,'a')");
+        Common.sqlExe(ds,"insert into label (id,user_id,label) values (2,1,'b')");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("label");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 2L);
+        data.put("user_id",1L);
+        data.put("label", "b");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("b;a_", response.getSource().get("_labels"));
+    }
+
+    /**
+     * 带函数子查询从表更新
+     */
+    @Test
+    public void test02() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"update label set label='aa' where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("label");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("user_id",1L);
+        data.put("label", "aa");
+        dml.setData(dataList);
+
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("label", "v");
+        dml.setOld(oldList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
+    }
+
+    /**
+     * 带函数子查询从表删除
+     */
+    @Test
+    public void test03() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"delete from label where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("DELETE");
+        dml.setDatabase("mytest");
+        dml.setTable("label");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("user_id",1L);
+        data.put("label", "a");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("b_", response.getSource().get("_labels"));
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java

@@ -0,0 +1,122 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.util.*;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+import javax.sql.DataSource;
+
+public class LabelSyncJoinSubTest {
+
+    private ESAdapter esAdapter;
+
+    @Before
+    public void init() {
+        AdapterConfigs.put("es", "mytest_user_join_sub.yml");
+        esAdapter = Common.init();
+    }
+
+    /**
+     * 子查询从表插入
+     */
+    @Test
+    public void test01() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"delete from label where id=1 or id=2");
+        Common.sqlExe(ds,"insert into label (id,user_id,label) values (1,1,'a')");
+        Common.sqlExe(ds,"insert into label (id,user_id,label) values (2,1,'b')");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("label");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 2L);
+        data.put("user_id",1L);
+        data.put("label", "b");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("b;a", response.getSource().get("_labels"));
+    }
+
+    /**
+     * 子查询从表更新
+     */
+    @Test
+    public void test02() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"update label set label='aa' where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("label");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("user_id",1L);
+        data.put("label", "aa");
+        dml.setData(dataList);
+
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("label", "a");
+        dml.setOld(oldList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("b;aa", response.getSource().get("_labels"));
+    }
+
+    /**
+     * 子查询从表删除
+     */
+    @Test
+    public void test03() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"delete from label where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("DELETE");
+        dml.setDatabase("mytest");
+        dml.setTable("label");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("user_id",1L);
+        data.put("label", "a");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("b", response.getSource().get("_labels"));
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java

@@ -0,0 +1,90 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.util.*;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+import javax.sql.DataSource;
+
+public class RoleSyncJoinOne2Test {
+
+    private ESAdapter esAdapter;
+
+    @Before
+    public void init() {
+        AdapterConfigs.put("es", "mytest_user_join_one2.yml");
+        esAdapter = Common.init();
+    }
+
+    /**
+     * 带函数非子查询从表插入
+     */
+    @Test
+    public void test01() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"delete from role where id=1");
+        Common.sqlExe(ds,"insert into role (id,role_name) values (1,'admin')");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("role");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("role_name", "admin");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("admin_", response.getSource().get("_role_name"));
+    }
+
+    /**
+     * 带函数非子查询从表更新
+     */
+    @Test
+    public void test02() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"update role set role_name='admin3' where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("role");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("role_name", "admin3");
+        dml.setData(dataList);
+
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("role_name", "admin");
+        dml.setOld(oldList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("admin3_", response.getSource().get("_role_name"));
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java

@@ -0,0 +1,174 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.util.*;
+
+import javax.sql.DataSource;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+public class RoleSyncJoinOneTest {
+
+    private ESAdapter esAdapter;
+
+    @Before
+    public void init() {
+        AdapterConfigs.put("es", "mytest_user_join_one.yml");
+        esAdapter = Common.init();
+    }
+
+    /**
+     * 非子查询从表插入
+     */
+    @Test
+    public void test01() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds, "delete from role where id=1");
+        Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("role");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("role_name", "admin");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("admin", response.getSource().get("_role_name"));
+    }
+
+    /**
+     * 非子查询从表更新
+     */
+    @Test
+    public void test02() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds, "update role set role_name='admin2' where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("role");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("role_name", "admin2");
+        dml.setData(dataList);
+
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("role_name", "admin");
+        dml.setOld(oldList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("admin2", response.getSource().get("_role_name"));
+    }
+
+    /**
+     * 主表更新外键值
+     */
+    @Test
+    public void test03() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds, "delete from role where id=2");
+        Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')");
+        Common.sqlExe(ds, "update user set role_id=2 where id=1");
+
+         Dml dml = new Dml();
+         dml.setDestination("example");
+         dml.setTs(new Date().getTime());
+         dml.setType("UPDATE");
+         dml.setDatabase("mytest");
+         dml.setTable("user");
+         List<Map<String, Object>> dataList = new ArrayList<>();
+         Map<String, Object> data = new LinkedHashMap<>();
+         dataList.add(data);
+         data.put("id", 1L);
+         data.put("role_id", 2L);
+         dml.setData(dataList);
+         List<Map<String, Object>> oldList = new ArrayList<>();
+         Map<String, Object> old = new LinkedHashMap<>();
+         oldList.add(old);
+         old.put("role_id", 1L);
+         dml.setOld(oldList);
+         esAdapter.getEsSyncService().sync(dml);
+
+         GetResponse response =
+         esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+         Assert.assertEquals("operator", response.getSource().get("_role_name"));
+
+        Common.sqlExe(ds, "update user set role_id=1 where id=1");
+
+        Dml dml2 = new Dml();
+        dml2.setDestination("example");
+        dml2.setTs(new Date().getTime());
+        dml2.setType("UPDATE");
+        dml2.setDatabase("mytest");
+        dml2.setTable("user");
+        List<Map<String, Object>> dataList2 = new ArrayList<>();
+        Map<String, Object> data2 = new LinkedHashMap<>();
+        dataList2.add(data2);
+        data2.put("id", 1L);
+        data2.put("role_id", 1L);
+        dml2.setData(dataList2);
+        List<Map<String, Object>> oldList2 = new ArrayList<>();
+        Map<String, Object> old2 = new LinkedHashMap<>();
+        oldList2.add(old2);
+        old2.put("role_id", 2L);
+        dml2.setOld(oldList2);
+        esAdapter.getEsSyncService().sync(dml2);
+
+        GetResponse response2 = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
+    }
+
+    /**
+     * 非子查询从表删除
+     */
+    @Test
+    public void test04() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds, "delete from role where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("DELETE");
+        dml.setDatabase("mytest");
+        dml.setTable("role");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("role_name", "admin");
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertNull(response.getSource().get("_role_name"));
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java

@@ -0,0 +1,91 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.util.*;
+
+import javax.sql.DataSource;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+public class UserSyncJoinOneTest {
+
+    private ESAdapter esAdapter;
+
+    @Before
+    public void init() {
+        AdapterConfigs.put("es", "mytest_user_join_one.yml");
+        esAdapter = Common.init();
+    }
+
+    /**
+     * 主表带函数插入
+     */
+    @Test
+    public void test01() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"delete from user where id=1");
+        Common.sqlExe(ds,"insert into user (id,name,role_id) values (1,'Eric',1)");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric");
+        data.put("role_id", 1L);
+        data.put("c_time", new Date());
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("Eric_", response.getSource().get("_name"));
+    }
+
+    /**
+     * 主表带函数更新
+     */
+    @Test
+    public void test02() {
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
+        Common.sqlExe(ds,"update user set name='Eric2' where id=1");
+
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric2");
+        dml.setData(dataList);
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("name", "Eric");
+        dml.setOld(oldList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("Eric2_", response.getSource().get("_name"));
+    }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java

@@ -0,0 +1,115 @@
+package com.alibaba.otter.canal.client.adapter.es.test.sync;
+
+import java.util.*;
+
+import org.elasticsearch.action.get.GetResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+public class UserSyncSingleTest {
+
+    private ESAdapter esAdapter;
+
+    @Before
+    public void init() {
+        AdapterConfigs.put("es", "mytest_user_single.yml");
+        esAdapter = Common.init();
+    }
+
+    /**
+     * 单表插入
+     */
+    @Test
+    public void test01() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric");
+        data.put("role_id", 1L);
+        data.put("c_time", new Date());
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("Eric", response.getSource().get("_name"));
+    }
+
+    /**
+     * 单表更新
+     */
+    @Test
+    public void test02() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric2");
+        dml.setData(dataList);
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("name", "Eric");
+        dml.setOld(oldList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertEquals("Eric2", response.getSource().get("_name"));
+    }
+
+    /**
+     * 单表删除
+     */
+    @Test
+    public void test03() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("DELETE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric");
+        data.put("role_id", 1L);
+        data.put("c_time", new Date());
+
+        dml.setData(dataList);
+
+        esAdapter.getEsSyncService().sync(dml);
+
+        GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
+        Assert.assertNull(response.getSource());
+    }
+
+    // @After
+    // public void after() {
+    // esAdapter.destroy();
+    // DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
+    // }
+}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql

@@ -0,0 +1,39 @@
+-- ----------------------------
+-- Table structure for label
+-- ----------------------------
+DROP TABLE IF EXISTS `label`;
+CREATE TABLE `label` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `user_id` bigint(20) NOT NULL,
+  `label` varchar(30) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Table structure for role
+-- ----------------------------
+DROP TABLE IF EXISTS `role`;
+CREATE TABLE `role` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `role_name` varchar(30) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Table structure for user
+-- ----------------------------
+DROP TABLE IF EXISTS `user`;
+CREATE TABLE `user` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `name` varchar(30) NOT NULL,
+  `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `role_id` bigint(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;
+
+insert into user (id,name,role_id) values (1,'Eric',1);
+insert into role (id,role_name) values (1,'admin');
+insert into role (id,role_name) values (2,'operator');
+insert into label (id,user_id,label) values (1,1,'a');
+insert into label (id,user_id,label) values (2,1,'b');
+commit;

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json

@@ -0,0 +1,21 @@
+{
+  "_doc": {
+    "properties": {
+      "_name": {
+        "type": "text"
+      },
+      "_role_id": {
+        "type": "long"
+      },
+      "_role_name": {
+        "type": "text"
+      },
+      "_labels": {
+        "type": "text"
+      },
+      "_c_time": {
+        "type": "date"
+      }
+    }
+  }
+}

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml

@@ -0,0 +1,10 @@
+dataSourceKey: defaultDS
+destination: example
+esMapping:
+  _index: mytest_user
+  _type: _doc
+  _id: _id
+  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
+        b.role_name as _role_name, a.c_time as _c_time from user a
+        left join role b on b.id=a.role_id"
+  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one2.yml

@@ -0,0 +1,10 @@
+dataSourceKey: defaultDS
+destination: example
+esMapping:
+  _index: mytest_user
+  _type: _doc
+  _id: _id
+  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
+        concat(b.role_name,'_') as _role_name, a.c_time as _c_time from user a
+        left join role b on b.id=a.role_id"
+  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml

@@ -0,0 +1,11 @@
+dataSourceKey: defaultDS
+destination: example
+esMapping:
+  _index: mytest_user
+  _type: _doc
+  _id: _id
+  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
+        b.labels _labels, a.c_time as _c_time from user a
+        left join (select user_id, group_concat(label order by id desc separator ';') as labels from label
+        group by user_id) b on b.user_id=a.id"
+  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub2.yml

@@ -0,0 +1,11 @@
+dataSourceKey: defaultDS
+destination: example
+esMapping:
+  _index: mytest_user
+  _type: _doc
+  _id: _id
+  sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
+        concat(b.labels, '_') as _labels, a.c_time as _c_time from user a
+        left join (select user_id, group_concat(label order by id desc separator ';') as labels from label
+        group by user_id) b on b.user_id=a.id"
+  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml

@@ -0,0 +1,8 @@
+dataSourceKey: defaultDS
+destination: example
+esMapping:
+  _index: mytest_user
+  _type: _doc
+  _id: _id
+  sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a"
+  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/log4j2-test.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="ERROR">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>

client-adapter/elasticsearch/src/test/resources/logback-test.xml

@@ -0,0 +1,13 @@
+<configuration scan="true" scanPeriod=" 5 seconds">
+	<jmxConfigurator />
+	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+			</pattern>
+		</encoder>
+	</appender>
+
+	<root level="TRACE">
+		<appender-ref ref="STDOUT"/>
+	</root>
+</configuration>

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java

@@ -53,8 +53,8 @@ public class HbaseAdapter implements OuterAdapter {
                         hbaseMapping = MappingConfigLoader.load();
                         mappingConfigCache = new HashMap<>();
                         for (MappingConfig mappingConfig : hbaseMapping.values()) {
-                            mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseMapping().getDestination())
+                            mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
-                                                   + "." + mappingConfig.getHbaseMapping().getDatabase() + "."
+                                                   + mappingConfig.getHbaseMapping().getDatabase() + "."
                                                    + mappingConfig.getHbaseMapping().getTable(),
                                 mappingConfig);
                         }
@@ -62,10 +62,10 @@ public class HbaseAdapter implements OuterAdapter {
                 }
             }
 
-            Map<String, String> propertites = configuration.getProperties();
+            Map<String, String> properties = configuration.getProperties();
 
             Configuration hbaseConfig = HBaseConfiguration.create();
-            propertites.forEach(hbaseConfig::set);
+            properties.forEach(hbaseConfig::set);
             conn = ConnectionFactory.createConnection(hbaseConfig);
             hbaseTemplate = new HbaseTemplate(conn);
             hbaseSyncService = new HbaseSyncService(hbaseTemplate);
@@ -100,33 +100,34 @@ public class HbaseAdapter implements OuterAdapter {
                 return etlResult;
             }
         } else {
-            DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(task);
+            StringBuilder resultMsg = new StringBuilder();
-            if (dataSource != null) {
+            boolean resSucc = true;
-                StringBuilder resultMsg = new StringBuilder();
+            // ds不为空说明传入的是datasourceKey
-                boolean resSucc = true;
+            for (MappingConfig configTmp : hbaseMapping.values()) {
-                // ds不为空说明传入的是datasourceKey
+                // 取所有的destination为task的配置
-                for (MappingConfig configTmp : hbaseMapping.values()) {
+                if (configTmp.getDestination().equals(task)) {
-                    // 取所有的datasourceKey为task的配置
+                    DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(configTmp.getDataSourceKey());
-                    if (configTmp.getDataSourceKey().equals(task)) {
+                    if (dataSource == null) {
-                        EtlResult etlRes = HbaseEtlService.importData(dataSource, hbaseTemplate, configTmp, params);
+                        continue;
-                        if (!etlRes.getSucceeded()) {
-                            resSucc = false;
-                            resultMsg.append(etlRes.getErrorMessage()).append("\n");
-                        } else {
-                            resultMsg.append(etlRes.getResultMessage()).append("\n");
-                        }
                     }
-                }
+                    EtlResult etlRes = HbaseEtlService.importData(dataSource, hbaseTemplate, configTmp, params);
-                if (resultMsg.length() > 0) {
+                    if (!etlRes.getSucceeded()) {
-                    etlResult.setSucceeded(resSucc);
+                        resSucc = false;
-                    if (resSucc) {
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
-                        etlResult.setResultMessage(resultMsg.toString());
                     } else {
-                        etlResult.setErrorMessage(resultMsg.toString());
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
                     }
-                    return etlResult;
                 }
             }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSucc);
+                if (resSucc) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
         }
         etlResult.setSucceeded(false);
         etlResult.setErrorMessage("Task not found");
@@ -170,7 +171,7 @@ public class HbaseAdapter implements OuterAdapter {
     public String getDestination(String task) {
         MappingConfig config = hbaseMapping.get(task);
         if (config != null && config.getHbaseMapping() != null) {
-            return config.getHbaseMapping().getDestination();
+            return config.getDestination();
         }
         return null;
     }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java

@@ -12,6 +12,8 @@ public class MappingConfig {
 
     private String       dataSourceKey; // 数据源key
 
+    private String       destination;   // canal实例或MQ的topic
+
     private HbaseMapping hbaseMapping;  // hbase映射配置
 
     public String getDataSourceKey() {
@@ -22,6 +24,14 @@ public class MappingConfig {
         this.dataSourceKey = dataSourceKey;
     }
 
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
     public HbaseMapping getHbaseMapping() {
         return hbaseMapping;
     }
@@ -143,7 +153,6 @@ public class MappingConfig {
     public static class HbaseMapping {
 
         private Mode                    mode               = Mode.STRING;           // hbase默认转换格式
-        private String                  destination;                                // canal实例或MQ的topic
         private String                  database;                                   // 数据库名或schema名
         private String                  table;                                      // 表面名
         private String                  hbaseTable;                                 // hbase表名
@@ -169,14 +178,6 @@ public class MappingConfig {
             this.mode = mode;
         }
 
-        public String getDestination() {
-            return destination;
-        }
-
-        public void setDestination(String destination) {
-            this.destination = destination;
-        }
-
         public String getDatabase() {
             return database;
         }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java

@@ -38,6 +38,9 @@ public class MappingConfigLoader {
         Map<String, MappingConfig> result = new LinkedHashMap<>();
 
         Collection<String> configs = AdapterConfigs.get("hbase");
+        if (configs == null) {
+            return result;
+        }
         for (String c : configs) {
             if (c == null) {
                 continue;

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java

@@ -38,8 +38,7 @@ public class HbaseSyncService {
                     delete(config, dml);
                 }
                 if (logger.isDebugEnabled()) {
-                    String res = dml.toString();
+                    logger.debug("DML: {}", dml.toString());
-                    logger.debug(res);
                 }
             }
         } catch (Exception e) {
@@ -199,7 +198,8 @@ public class HbaseSyncService {
             Map<String, MappingConfig.ColumnItem> columnItems = hbaseMapping.getColumnItems();
             HRow hRow = new HRow(rowKeyBytes);
             for (String updateColumn : old.get(index).keySet()) {
-                if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(updateColumn)) {
+                if (hbaseMapping.getExcludeColumns() != null
+                    && hbaseMapping.getExcludeColumns().contains(updateColumn)) {
                     continue;
                 }
                 MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);

client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml

@@ -1,7 +1,7 @@
 dataSourceKey: defaultDS
+destination: example
 hbaseMapping:
   mode: PHOENIX  #NATIVE   #STRING
-  destination: example
   database: mytest  # 数据库名
   table: person2     # 数据库表名
   hbaseTable: MYTEST.PERSON2   # HBase表名

client-adapter/launcher/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
@@ -32,7 +33,7 @@
         <dependency>
             <groupId>com.alibaba.otter</groupId>
             <artifactId>canal.client</artifactId>
-            <version>1.1.2-SNAPSHOT</version>
+            <version>${canal_version}</version>
         </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
@@ -93,6 +94,19 @@
             <classifier>jar-with-dependencies</classifier>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.elasticsearch</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 
     <build>
@@ -104,13 +118,17 @@
                 <version>2.0.1.RELEASE</version>
                 <configuration>
                     <excludes>
+                        <exclude>
+                            <groupId>com.alibaba.otter</groupId>
+                            <artifactId>client-adapter.logger</artifactId>
+                        </exclude>
                         <exclude>
                             <groupId>com.alibaba.otter</groupId>
                             <artifactId>client-adapter.hbase</artifactId>
                         </exclude>
                         <exclude>
                             <groupId>com.alibaba.otter</groupId>
-                            <artifactId>client-adapter.logger</artifactId>
+                            <artifactId>client-adapter.elasticsearch</artifactId>
                         </exclude>
                     </excludes>
                 </configuration>
@@ -134,11 +152,17 @@
                             <tasks>
                                 <copy todir="${project.basedir}/target/config" overwrite="true">
                                     <fileset dir="${project.basedir}/src/main/resources" erroronmissingdir="true">
-                                        <include name="*.yml" />
+                                        <include name="*.yml"/>
                                     </fileset>
                                 </copy>
                                 <copy todir="${project.basedir}/target/config/hbase" overwrite="true">
-                                    <fileset dir="${project.basedir}/../hbase/src/main/resources/hbase" erroronmissingdir="true">
+                                    <fileset dir="${project.basedir}/../hbase/src/main/resources/hbase"
+                                             erroronmissingdir="true">
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/target/config/es" overwrite="true">
+                                    <fileset dir="${project.basedir}/../elasticsearch/src/main/resources/es" erroronmissingdir="true">
                                         <include name="*.yml" />
                                     </fileset>
                                 </copy>
@@ -201,6 +225,14 @@
                         </executions>
                     </plugin>
                 </plugins>
+                <resources>
+                    <resource>
+                        <directory>src/main/resources</directory>
+                        <excludes>
+                            <exclude>application.yml</exclude>
+                        </excludes>
+                    </resource>
+                </resources>
             </build>
         </profile>
     </profiles>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java

@@ -2,11 +2,10 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalMQConnector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -117,6 +116,54 @@ public abstract class AbstractCanalAdapterWorker {
         });
     }
 
+    protected void mqWriteOutData(int retry, long timeout, boolean flatMessage, CanalMQConnector connector,
+                        ExecutorService workerExecutor) {
+        for (int i = 0; i < retry; i++) {
+            try {
+                List<?> messages;
+                if (!flatMessage) {
+                    messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
+                } else {
+                    messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
+                }
+                if (messages != null) {
+                    Future<Boolean> future = workerExecutor.submit(() -> {
+                        for (final Object message : messages) {
+                            if (message instanceof FlatMessage) {
+                                writeOut((FlatMessage) message);
+                            } else {
+                                writeOut((Message) message);
+                            }
+                        }
+                        return true;
+                    });
+
+                    try {
+                        future.get(timeout, TimeUnit.MILLISECONDS);
+                    } catch (Exception e) {
+                        future.cancel(true);
+                        throw e;
+                    }
+                }
+                connector.ack();
+                break;
+            } catch (Throwable e) {
+                if (i == retry - 1) {
+                    connector.ack();
+                } else {
+                    connector.rollback();
+                }
+
+                logger.error(e.getMessage(), e);
+                try {
+                    TimeUnit.SECONDS.sleep(1L);
+                } catch (InterruptedException e1) {
+                    // ignore
+                }
+            }
+        }
+    }
+
     public void start() {
         if (!running) {
             thread = new Thread(this::process);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java

@@ -1,16 +1,14 @@
 package com.alibaba.otter.canal.adapter.launcher.loader;
 
 import java.util.List;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
-import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.common.errors.WakeupException;
 
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
 import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
-import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
-import com.alibaba.otter.canal.protocol.FlatMessage;
-import com.alibaba.otter.canal.protocol.Message;
 
 /**
  * kafka对应的client适配器工作线程
@@ -20,17 +18,24 @@ import com.alibaba.otter.canal.protocol.Message;
  */
 public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
 
+    private CanalClientConfig   canalClientConfig;
     private KafkaCanalConnector connector;
     private String              topic;
     private boolean             flatMessage;
 
-    public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
+    public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
-                                   List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
+                                   String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
         super(canalOuterAdapters);
+        this.canalClientConfig = canalClientConfig;
         this.topic = topic;
         this.canalDestination = topic;
         this.flatMessage = flatMessage;
-        this.connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
+        this.connector = new KafkaCanalConnector(bootstrapServers,
+            topic,
+            null,
+            groupId,
+            canalClientConfig.getBatchSize(),
+            flatMessage);
         // connector.setSessionTimeout(1L, TimeUnit.MINUTES);
     }
 
@@ -38,6 +43,10 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
     protected void process() {
         while (!running)
             ;
+        ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
+        int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry();
+        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
+
         while (running) {
             try {
                 syncSwitch.get(canalDestination);
@@ -47,35 +56,12 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
                 connector.subscribe();
                 logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
                 while (running) {
-                    try {
+                    Boolean status = syncSwitch.status(canalDestination);
-                        Boolean status = syncSwitch.status(canalDestination);
+                    if (status != null && !status) {
-                        if (status != null && !status) {
+                        connector.disconnect();
-                            connector.disconnect();
+                        break;
-                            break;
-                        }
-
-                        List<?> messages;
-                        if (!flatMessage) {
-                            messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
-                        } else {
-                            messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
-                        }
-                        if (messages != null) {
-                            for (final Object message : messages) {
-                                if (message instanceof FlatMessage) {
-                                    writeOut((FlatMessage) message);
-                                } else {
-                                    writeOut((Message) message);
-                                }
-                            }
-                        }
-                        connector.ack();
-                    } catch (CommitFailedException e) {
-                        logger.warn(e.getMessage());
-                    } catch (Throwable e) {
-                        logger.error(e.getMessage(), e);
-                        TimeUnit.SECONDS.sleep(1L);
                     }
+                    mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor);
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -67,9 +67,15 @@ public class CanalAdapterLoader {
                 }
                 CanalAdapterWorker worker;
                 if (sa != null) {
-                    worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups);
+                    worker = new CanalAdapterWorker(canalClientConfig,
+                        instance.getInstance(),
+                        sa,
+                        canalOuterAdapterGroups);
                 } else if (zkHosts != null) {
-                    worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups);
+                    worker = new CanalAdapterWorker(canalClientConfig,
+                        instance.getInstance(),
+                        zkHosts,
+                        canalOuterAdapterGroups);
                 } else {
                     throw new RuntimeException("No canal server connector found");
                 }
@@ -90,7 +96,8 @@ public class CanalAdapterLoader {
                     }
                     canalOuterAdapterGroups.add(canalOuterAdapters);
                     if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
-                        CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig.getBootstrapServers(),
+                        CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
+                            canalClientConfig.getBootstrapServers(),
                             topic.getTopic(),
                             group.getGroupId(),
                             canalOuterAdapterGroups,
@@ -98,7 +105,8 @@ public class CanalAdapterLoader {
                         canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
                         rocketMQWorker.start();
                     } else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
-                        CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig.getBootstrapServers(),
+                        CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
+                            canalClientConfig.getBootstrapServers(),
                             topic.getTopic(),
                             group.getGroupId(),
                             canalOuterAdapterGroups,

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java

@@ -1,13 +1,16 @@
 package com.alibaba.otter.canal.adapter.launcher.loader;
 
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
 import org.apache.kafka.common.errors.WakeupException;
 
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
 import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
-import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectors;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
@@ -18,17 +21,19 @@ import com.alibaba.otter.canal.protocol.Message;
  */
 public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
 
+    private CanalClientConfig      canalClientConfig;
     private RocketMQCanalConnector connector;
     private String                 topic;
     private boolean                flatMessage;
 
-    public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId,
+    public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
-                                      List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
+                                      String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
         super(canalOuterAdapters);
+        this.canalClientConfig = canalClientConfig;
         this.topic = topic;
         this.flatMessage = flatMessage;
         this.canalDestination = topic;
-        this.connector = RocketMQCanalConnectors.newRocketMQConnector(nameServers, topic, groupId, flatMessage);
+        this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage);
         logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
     }
 
@@ -36,6 +41,11 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
     protected void process() {
         while (!running)
             ;
+
+        ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
+        int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry();
+        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
+
         while (running) {
             try {
                 syncSwitch.get(canalDestination);
@@ -45,33 +55,12 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
                 connector.subscribe();
                 logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
                 while (running) {
-                    try {
+                    Boolean status = syncSwitch.status(canalDestination);
-                        Boolean status = syncSwitch.status(canalDestination);
+                    if (status != null && !status) {
-                        if (status != null && !status) {
+                        connector.disconnect();
-                            connector.disconnect();
+                        break;
-                            break;
-                        }
-
-                        List<?> messages;
-                        if (!flatMessage) {
-                            messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
-                        } else {
-                            messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
-                        }
-                        if (messages != null) {
-                            for (final Object message : messages) {
-                                if (message instanceof FlatMessage) {
-                                    writeOut((FlatMessage) message);
-                                } else {
-                                    writeOut((Message) message);
-                                }
-                            }
-                        }
-                        connector.ack();
-                    } catch (Throwable e) {
-                        logger.error(e.getMessage(), e);
-                        TimeUnit.SECONDS.sleep(1L);
                     }
+                    mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor);
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java

@@ -2,12 +2,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
 
 import java.net.SocketAddress;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
-import java.util.concurrent.TimeoutException;
 
 import com.alibaba.otter.canal.client.CanalConnector;
 import com.alibaba.otter.canal.client.CanalConnectors;
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
 import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
 import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
 import com.alibaba.otter.canal.protocol.Message;
@@ -20,10 +20,12 @@ import com.alibaba.otter.canal.protocol.Message;
  */
 public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
 
-    private static final int BATCH_SIZE = 50;
+    private static final int  BATCH_SIZE = 50;
-    private static final int SO_TIMEOUT = 0;
+    private static final int  SO_TIMEOUT = 0;
 
-    private CanalConnector   connector;
+    private CanalConnector    connector;
+
+    private CanalClientConfig canalClientConfig;
 
     /**
      * 单台client适配器worker的构造方法
@@ -32,9 +34,10 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
      * @param address canal-server地址
      * @param canalOuterAdapters 外部适配器组
      */
-    public CanalAdapterWorker(String canalDestination, SocketAddress address,
+    public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
                               List<List<OuterAdapter>> canalOuterAdapters){
         super(canalOuterAdapters);
+        this.canalClientConfig = canalClientConfig;
         this.canalDestination = canalDestination;
         connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
     }
@@ -46,10 +49,11 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
      * @param zookeeperHosts zookeeper地址
      * @param canalOuterAdapters 外部适配器组
      */
-    public CanalAdapterWorker(String canalDestination, String zookeeperHosts,
+    public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
                               List<List<OuterAdapter>> canalOuterAdapters){
         super(canalOuterAdapters);
         this.canalDestination = canalDestination;
+        this.canalClientConfig = canalClientConfig;
         connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
         ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
     }
@@ -58,6 +62,15 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
     protected void process() {
         while (!running)
             ; // waiting until running == true
+
+        ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
+        int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry();
+        long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟
+        Integer batchSize = canalClientConfig.getBatchSize();
+        if (batchSize == null) {
+            batchSize = BATCH_SIZE;
+        }
+
         while (running) {
             try {
                 syncSwitch.get(canalDestination);
@@ -77,35 +90,50 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
                         break;
                     }
 
-                    // server配置canal.instance.network.soTimeout(默认: 30s)
+                    for (int i = 0; i < retry; i++) {
-                    // 范围内未与server交互，server将关闭本次socket连接
+                        Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
-                    Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据
+                        long batchId = message.getId();
-                    long batchId = message.getId();
+                        try {
-                    try {
+                            int size = message.getEntries().size();
-                        int size = message.getEntries().size();
+                            if (batchId == -1 || size == 0) {
-                        if (batchId == -1 || size == 0) {
+                                Thread.sleep(500);
-                            Thread.sleep(1000);
+                            } else {
-                        } else {
+                                Future<Boolean> future = workerExecutor.submit(() -> {
-                            if (logger.isDebugEnabled()) {
+                                    if (logger.isDebugEnabled()) {
-                                logger.debug("destination: {} batchId: {} batchSize: {} ",
+                                        logger.debug("destination: {} batchId: {} batchSize: {} ",
-                                    this.canalDestination,
+                                            canalDestination,
-                                    batchId,
+                                            batchId,
-                                    size);
+                                            size);
+                                    }
+                                    long begin = System.currentTimeMillis();
+                                    writeOut(message);
+                                    if (logger.isDebugEnabled()) {
+                                        logger.debug("destination: {} batchId: {} elapsed time: {} ms",
+                                            canalDestination,
+                                            batchId,
+                                            System.currentTimeMillis() - begin);
+                                    }
+                                    return true;
+                                });
+
+                                try {
+                                    future.get(timeout, TimeUnit.MILLISECONDS);
+                                } catch (Exception e) {
+                                    future.cancel(true);
+                                    throw e;
+                                }
                             }
-                            long begin = System.currentTimeMillis();
+                            connector.ack(batchId); // 提交确认
-                            writeOut(message);
+                            break;
-                            if (logger.isDebugEnabled()) {
+                        } catch (Exception e) {
-                                logger.debug("destination: {} batchId: {} elapsed time: {} ms",
+                            if (i != retry - 1) {
-                                    this.canalDestination,
+                                connector.rollback(batchId); // 处理失败, 回滚数据
-                                    batchId,
+                            } else {
-                                    System.currentTimeMillis() - begin);
+                                connector.ack(batchId);
                             }
+                            logger.error("sync error!", e);
+                            Thread.sleep(500);
                         }
-                        connector.ack(batchId); // 提交确认
-                    } catch (Exception e) {
-                        connector.rollback(batchId); // 处理失败, 回滚数据
-                        logger.error("sync error!", e);
-                        Thread.sleep(500);
                     }
                 }
 
@@ -124,6 +152,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
                 }
             }
         }
+
+        workerExecutor.shutdown();
     }
 
     @Override

client-adapter/launcher/src/main/resources/application.yml

@@ -3,32 +3,32 @@ server:
 logging:
   level:
     com.alibaba.otter.canal.client.adapter.hbase: DEBUG
+    com.alibaba.otter.canal.client.adapter.es: TRACE
 spring:
   jackson:
     date-format: yyyy-MM-dd HH:mm:ss
     time-zone: GMT+8
     default-property-inclusion: non_null
 
-
+#canal.conf:
-hbasezookeeper.quorum: 127.0.0.1
+#  canalServerHost: 127.0.0.1:11111
-hbase.zookeeper.property.clientPort: 2181
-hbase.zookeeper.znode.parent: /hbase
-
-canal.conf:
-  canalServerHost: 127.0.0.1:11111
 #  zookeeperHosts: slave1:2181
-#  bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876
+#  bootstrapServers: slave1:6667 #or rocketmq
-  flatMessage: true
+#  flatMessage: true
-  canalInstances:
+#  canalInstances:
-  - instance: example
+#  - instance: example
-    groups:
+#    groups:
-    - outAdapters:
+#    - outAdapters:
-      - name: logger
+#      - name: logger
 #      - name: hbase
 #        properties:
-#          hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
+#          hbase.zookeeper.quorum: 127.0.0.1
-#          hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
+#          hbase.zookeeper.property.clientPort: 2181
-#          zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
+#          zookeeper.znode.parent: /hbase
+#      - name: es
+#        hosts: 127.0.0.1:9300
+#        properties:
+#          cluster.name: elasticsearch
 #  mqTopics:
 #  - mqMode: kafka
 #    topic: example
@@ -52,3 +52,4 @@ canal.conf:
 #      password: 121212
 #  adapterConfigs:
 #  - hbase/mytest_person2.yml
+#  - es/mytest_user.yml

client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java

@@ -27,7 +27,7 @@ public class LoggerAdapterExample implements OuterAdapter {
 
     @Override
     public void sync(Dml dml) {
-        logger.info(dml.toString());
+        logger.info("DML: {}", dml.toString());
     }
 
     @Override

client-adapter/pom.xml

@@ -14,13 +14,15 @@
         <java_source_version>1.8</java_source_version>
         <java_target_version>1.8</java_target_version>
         <file_encoding>UTF-8</file_encoding>
-        <canal_version>1.1.1-SNAPSHOT</canal_version>
+
+        <canal_version>1.1.2-SNAPSHOT</canal_version>
     </properties>
 
     <modules>
         <module>common</module>
         <module>logger</module>
         <module>hbase</module>
+        <module>elasticsearch</module>
         <module>launcher</module>
     </modules>
 

client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java

@@ -32,10 +32,12 @@ public class CanalMessageDeserializer {
                         if (lazyParseEntry) {
                             // byteString
                             result.setRawEntries(messages.getMessagesList());
+                            result.setRaw(true);
                         } else {
                             for (ByteString byteString : messages.getMessagesList()) {
                                 result.addEntry(CanalEntry.Entry.parseFrom(byteString));
                             }
+                            result.setRaw(false);
                         }
                         return result;
                     }

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java

@@ -133,7 +133,7 @@ public class SimpleCanalConnector implements CanalConnector {
                 runningMonitor.stop();
             }
         } else {
-            doDisconnnect();
+            doDisconnect();
         }
     }
 
@@ -190,7 +190,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private void doDisconnnect() throws CanalClientException {
+    private void doDisconnect() throws CanalClientException {
         if (readableChannel != null) {
             quietlyClose(readableChannel);
             readableChannel = null;
@@ -434,7 +434,7 @@ public class SimpleCanalConnector implements CanalConnector {
 
                 public void processActiveExit() {
                     mutex.set(false);
-                    doDisconnnect();
+                    doDisconnect();
                 }
 
             });

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -42,7 +42,8 @@ public class KafkaCanalConnector implements CanalMQConnector {
     private volatile boolean               running   = false;
     private boolean                        flatMessage;
 
-    public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, boolean flatMessage){
+    public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
+                               boolean flatMessage){
         this.topic = topic;
         this.partition = partition;
         this.flatMessage = flatMessage;
@@ -55,7 +56,10 @@ public class KafkaCanalConnector implements CanalMQConnector {
         properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
         properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
         properties.put("session.timeout.ms", "30000"); // 默认为30秒
-        properties.put("max.poll.records", "100");
+        if (batchSize == null) {
+            batchSize = 100;
+        }
+        properties.put("max.poll.records", batchSize.toString());
         properties.put("key.deserializer", StringDeserializer.class.getName());
         if (!flatMessage) {
             properties.put("value.deserializer", MessageDeserializer.class.getName());

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java

@@ -1,50 +0,0 @@
-package com.alibaba.otter.canal.client.kafka;
-
-/**
- * canal kafka connectors创建工具类
- *
- * @author machengyuan @ 2018-6-12
- * @version 1.0.0
- */
-public class KafkaCanalConnectors {
-
-    /**
-     * 创建kafka客户端链接，独立运行不注册zk信息
-     *
-     * @param servers
-     * @param topic
-     * @param partition
-     * @param groupId
-     * @return
-     */
-    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) {
-        return new KafkaCanalConnector(servers, topic, partition, groupId, false);
-    }
-
-    /**
-     * 创建kafka客户端链接，独立运行不注册zk信息
-     *
-     * @param servers
-     * @param topic
-     * @param groupId
-     * @return
-     */
-    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
-        return new KafkaCanalConnector(servers, topic, null, groupId, false);
-    }
-
-    /**
-     * 创建kafka客户端链接
-     *
-     * @param servers
-     * @param topic
-     * @param partition
-     * @param groupId
-     * @param flatMessage
-     * @return
-     */
-    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition,
-                                                        String groupId, boolean flatMessage) {
-        return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage);
-    }
-}

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java

@@ -1,24 +0,0 @@
-package com.alibaba.otter.canal.client.rocketmq;
-
-/**
- * RocketMQ connector provider.
- */
-public class RocketMQCanalConnectors {
-
-    /**
-     * Create RocketMQ connector
-     *
-     * @param nameServers name servers for RocketMQ
-     * @param topic
-     * @param groupId
-     * @return {@link RocketMQCanalConnector}
-     */
-    public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId) {
-        return new RocketMQCanalConnector(nameServers, topic, groupId, false);
-    }
-
-    public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId,
-                                                              boolean flatMessage) {
-        return new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage);
-    }
-}

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java

@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
 
 import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
-import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -36,12 +35,13 @@ public class CanalKafkaClientExample {
                                                     };
 
     public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
-        connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false);
+        connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
     }
 
     public static void main(String[] args) {
         try {
-            final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
+            final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
+                AbstractKafkaTest.zkServers,
                 AbstractKafkaTest.servers,
                 AbstractKafkaTest.topic,
                 AbstractKafkaTest.partition,

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java

@@ -11,7 +11,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
-import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -30,7 +29,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
     public void testKafkaConsumer() {
         final ExecutorService executor = Executors.newFixedThreadPool(1);
 
-        final KafkaCanalConnector connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId);
+        final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
 
         executor.submit(new Runnable() {
 

client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java

@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
 
 import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
-import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectors;
 import com.alibaba.otter.canal.client.running.kafka.AbstractKafkaTest;
 import com.alibaba.otter.canal.protocol.Message;
 
@@ -37,7 +36,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
                                                     };
 
     public CanalRocketMQClientExample(String nameServers, String topic, String groupId){
-        connector = RocketMQCanalConnectors.newRocketMQConnector(nameServers, topic, groupId);
+        connector = new RocketMQCanalConnector(nameServers, topic, groupId, false);
     }
 
     public static void main(String[] args) {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java

@@ -1,5 +1,9 @@
 package com.taobao.tddl.dbsync.binlog;
 
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2;
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4;
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr;
+
 /**
  * 处理下MySQL json二进制转化为可读的字符串
  * 
@@ -330,12 +334,30 @@ public class JsonConversion {
                             long ultime = Math.abs(packed_value);
                             long intpart = ultime >> 24;
                             int frac = (int) (ultime % (1L << 24));
-                            text = String.format("%s%02d:%02d:%02d",
+                            // text = String.format("%s%02d:%02d:%02d",
-                                packed_value >= 0 ? "" : "-",
+                            // packed_value >= 0 ? "" : "-",
-                                (int) ((intpart >> 12) % (1 << 10)),
+                            // (int) ((intpart >> 12) % (1 << 10)),
-                                (int) ((intpart >> 6) % (1 << 6)),
+                            // (int) ((intpart >> 6) % (1 << 6)),
-                                (int) (intpart % (1 << 6)));
+                            // (int) (intpart % (1 << 6)));
-                            text = text + "." + usecondsToStr(frac, 6);
+                            // text = text + "." + usecondsToStr(frac, 6);
+                            StringBuilder builder = new StringBuilder(17);
+                            if (packed_value < 0) {
+                                builder.append('-');
+                            }
+
+                            int d = (int) ((intpart >> 12) % (1 << 10));
+                            if (d > 100) {
+                                builder.append(String.valueOf(d));
+                            } else {
+                                appendNumber2(builder, d);
+                            }
+                            builder.append(':');
+                            appendNumber2(builder, (int) ((intpart >> 6) % (1 << 6)));
+                            builder.append(':');
+                            appendNumber2(builder, (int) (intpart % (1 << 6)));
+
+                            builder.append('.').append(usecondsToStr(frac, 6));
+                            text = builder.toString();
                         }
                         buf.append('"').append(text).append('"');
                     } else if (m_field_type == LogEvent.MYSQL_TYPE_DATE || m_field_type == LogEvent.MYSQL_TYPE_DATETIME
@@ -351,14 +373,28 @@ public class JsonConversion {
                             long ymd = intpart >> 17;
                             long ym = ymd >> 5;
                             long hms = intpart % (1 << 17);
-                            text = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                            // text =
-                                (int) (ym / 13),
+                            // String.format("%04d-%02d-%02d %02d:%02d:%02d",
-                                (int) (ym % 13),
+                            // (int) (ym / 13),
-                                (int) (ymd % (1 << 5)),
+                            // (int) (ym % 13),
-                                (int) (hms >> 12),
+                            // (int) (ymd % (1 << 5)),
-                                (int) ((hms >> 6) % (1 << 6)),
+                            // (int) (hms >> 12),
-                                (int) (hms % (1 << 6)));
+                            // (int) ((hms >> 6) % (1 << 6)),
-                            text = text + "." + usecondsToStr(frac, 6);
+                            // (int) (hms % (1 << 6)));
+                            StringBuilder builder = new StringBuilder(26);
+                            appendNumber4(builder, (int) (ym / 13));
+                            builder.append('-');
+                            appendNumber2(builder, (int) (ym % 13));
+                            builder.append('-');
+                            appendNumber2(builder, (int) (ymd % (1 << 5)));
+                            builder.append(' ');
+                            appendNumber2(builder, (int) (hms >> 12));
+                            builder.append(':');
+                            appendNumber2(builder, (int) ((hms >> 6) % (1 << 6)));
+                            builder.append(':');
+                            appendNumber2(builder, (int) (hms % (1 << 6)));
+                            builder.append('.').append(usecondsToStr(frac, 6));
+                            text = builder.toString();
                         }
                         buf.append('"').append(text).append('"');
                     } else {
@@ -397,22 +433,4 @@ public class JsonConversion {
         OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR
     }
 
-    private static String usecondsToStr(int frac, int meta) {
-        String sec = String.valueOf(frac);
-        if (meta > 6) {
-            throw new IllegalArgumentException("unknow useconds meta : " + meta);
-        }
-
-        if (sec.length() < 6) {
-            StringBuilder result = new StringBuilder(6);
-            int len = 6 - sec.length();
-            for (; len > 0; len--) {
-                result.append('0');
-            }
-            result.append(sec);
-            sec = result.toString();
-        }
-
-        return sec.substring(0, meta);
-    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java

@@ -0,0 +1,154 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
+import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_enum_type;
+
+/**
+ * 处理mysql8.0 parital json diff解析
+ * 
+ * @author agapple 2018年11月4日 下午3:53:46
+ * @since 1.1.2
+ */
+public class JsonDiffConversion {
+
+    /**
+     * The JSON value in the given path is replaced with a new value. It has the
+     * same effect as `JSON_REPLACE(col, path, value)`.
+     */
+    public static final int DIFF_OPERATION_REPLACE    = 0;
+    /**
+     * Add a new element at the given path. If the path specifies an array
+     * element, it has the same effect as `JSON_ARRAY_INSERT(col, path, value)`.
+     * If the path specifies an object member, it has the same effect as
+     * `JSON_INSERT(col, path, value)`.
+     */
+    public static final int DIFF_OPERATION_INSERT     = 1;
+    /**
+     * The JSON value at the given path is removed from an array or object. It
+     * has the same effect as `JSON_REMOVE(col, path)`.
+     */
+    public static final int DIFF_OPERATION_REMOVE     = 2;
+
+    public static final int JSON_DIFF_OPERATION_COUNT = 3;
+
+    public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
+                                                String charsetName) {
+        int position = buffer.position();
+        List<String> operation_names = new ArrayList<String>();
+        while (buffer.hasRemaining()) {
+            int operation_int = buffer.getUint8();
+            if (operation_int >= JSON_DIFF_OPERATION_COUNT) {
+                throw new IllegalArgumentException("reading operation type (invalid operation code)");
+            }
+
+            // skip path
+            long path_length = buffer.getPackedLong();
+            if (path_length > len) {
+                throw new IllegalArgumentException("skipping path");
+            }
+
+            // compute operation name
+            byte[] lastP = buffer.getData(buffer.position() + (int) path_length - 1, 1);
+            String operation_name = json_diff_operation_name(operation_int, lastP[0]);
+            operation_names.add(operation_name);
+
+            buffer.forward((int) path_length);
+            // skip value
+            if (operation_int != DIFF_OPERATION_REMOVE) {
+                long value_length = buffer.getPackedLong();
+                if (value_length > len) {
+                    throw new IllegalArgumentException("skipping path");
+                }
+
+                buffer.forward((int) value_length);
+            }
+        }
+
+        // Print function names in reverse order.
+        StringBuilder builder = new StringBuilder();
+        for (int i = operation_names.size() - 1; i >= 0; i--) {
+            if (i == 0 || operation_names.get(i - 1) != operation_names.get(i)) {
+                builder.append(operation_names.get(i)).append("(");
+            }
+        }
+
+        // Print column id
+        if (columnName != null) {
+            builder.append(columnName);
+        } else {
+            builder.append("@").append(columnIndex);
+        }
+
+        // In case this vector is empty (a no-op), make an early return
+        // after printing only the column name
+        if (operation_names.size() == 0) {
+            return builder;
+        }
+
+        // Print comma between column name and next function argument
+        builder.append(", ");
+        // Print paths and values.
+        buffer.position(position);
+        int diff_i = 0;
+        while (buffer.hasRemaining()) {
+            // Read operation
+            int operation_int = buffer.getUint8();
+
+            // Read path length
+            long path_length = buffer.getPackedLong();
+            // Print path
+            builder.append('\'').append(buffer.getFixString((int) path_length)).append('\'');
+
+            if (operation_int != DIFF_OPERATION_REMOVE) {
+                // Print comma between path and value
+                builder.append(", ");
+                // Read value length
+                long value_length = buffer.getPackedLong();
+
+                Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
+                    buffer,
+                    value_length - 1,
+                    charsetName);
+                buffer.forward((int) value_length - 1);
+                // Read value
+                if (jsonValue.m_type == Json_enum_type.ERROR) {
+                    throw new IllegalArgumentException("parsing json value");
+                }
+                StringBuilder jsonBuilder = new StringBuilder();
+                jsonValue.toJsonString(jsonBuilder, charsetName);
+                builder.append(jsonBuilder);
+            }
+
+            // Print closing parenthesis
+            if (!buffer.hasRemaining() || operation_names.get(diff_i + 1) != operation_names.get(diff_i)) {
+                builder.append(")");
+            }
+
+            if (buffer.hasRemaining()) {
+                builder.append(", ");
+            }
+            diff_i++;
+        }
+
+        return builder;
+    }
+
+    private static String json_diff_operation_name(int operationType, int last_path_char) {
+        switch (operationType) {
+            case DIFF_OPERATION_REPLACE:
+                return "JSON_REPLACE";
+            case DIFF_OPERATION_INSERT:
+                if (last_path_char == ']') {
+                    return "JSON_ARRAY_INSERT";
+                } else {
+                    return "JSON_INSERT";
+                }
+            case DIFF_OPERATION_REMOVE:
+                return "JSON_REMOVE";
+        }
+        throw new IllegalArgumentException("illeagal operationType : " + operationType);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java

@@ -1484,7 +1484,9 @@ public class LogBuffer {
 
         for (int bit = 0; bit < len; bit += 8) {
             int flag = ((int) buf[pos++]) & 0xff;
-            if (flag == 0) continue;
+            if (flag == 0) {
+                continue;
+            }
             if ((flag & 0x01) != 0) bitmap.set(bit);
             if ((flag & 0x02) != 0) bitmap.set(bit + 1);
             if ((flag & 0x04) != 0) bitmap.set(bit + 2);

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -372,6 +372,14 @@ public final class LogDecoder {
                 header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
+            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT: {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent, true);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
+                return event;
+            }
             case LogEvent.GTID_LOG_EVENT:
             case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
                 GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java

@@ -169,9 +169,15 @@ public abstract class LogEvent {
     /* Prepared XA transaction terminal event similar to Xid */
     public static final int    XA_PREPARE_LOG_EVENT                     = 38;
 
+    /**
+     * Extension of UPDATE_ROWS_EVENT, allowing partial values according to
+     * binlog_row_value_options.
+     */
+    public static final int    PARTIAL_UPDATE_ROWS_EVENT                = 39;
+
     // mariaDb 5.5.34
     /* New MySQL/Sun events are to be added right above this comment */
-    public static final int    MYSQL_EVENTS_END                         = 39;
+    public static final int    MYSQL_EVENTS_END                         = 49;
 
     public static final int    MARIA_EVENTS_BEGIN                       = 160;
     /* New Maria event numbers start from here */
@@ -361,6 +367,8 @@ public abstract class LogEvent {
                 return "Anonymous_Gtid";
             case PREVIOUS_GTIDS_LOG_EVENT:
                 return "Previous_gtids";
+            case PARTIAL_UPDATE_ROWS_EVENT:
+                return "Update_rows_partial";
             default:
                 return "Unknown"; /* impossible */
         }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java

@@ -211,6 +211,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[TRANSACTION_CONTEXT_EVENT - 1] = TRANSACTION_CONTEXT_HEADER_LEN;
                 postHeaderLen[VIEW_CHANGE_EVENT - 1] = VIEW_CHANGE_HEADER_LEN;
                 postHeaderLen[XA_PREPARE_LOG_EVENT - 1] = XA_PREPARE_HEADER_LEN;
+                postHeaderLen[PARTIAL_UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
 
                 // mariadb 10
                 postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java

@@ -1,6 +1,7 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.charset.Charset;
 
 import com.taobao.tddl.dbsync.binlog.CharsetConversion;
@@ -285,12 +286,12 @@ public class QueryLogEvent extends LogEvent {
                                                                   * type,
                                                                   * sql_mode
                                                                   */
-                                                         + 1 + 1 + 255 /*
+                                                         + 1 + 1 + 255/*
-                                                                        * type,
+                                                                       * type,
-                                                                        * length
+                                                                       * length
-                                                                        * ,
+                                                                       * ,
-                                                                        * catalog
+                                                                       * catalog
-                                                                        */
+                                                                       */
                                                          + 1 + 4 /*
                                                                   * type,
                                                                   * auto_increment
@@ -330,14 +331,32 @@ public class QueryLogEvent extends LogEvent {
                                                           * MariaDb type,
                                                           * sec_part of NOW()
                                                           */
-                                                         + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 32 * 3
+                                                         + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 /*
-                                                         + 1 + 60/*
+                                                                                                            * type
-                                                                  * type ,
+                                                                                                            * ,
-                                                                  * user_len ,
+                                                                                                            * microseconds
-                                                                  * user ,
+                                                                                                            */+ 1 + 32
-                                                                  * host_len ,
+                                                         * 3 + 1 + 60/*
-                                                                  * host
+                                                                      * type ,
-                                                                  */);
+                                                                      * user_len
+                                                                      * , user ,
+                                                                      * host_len
+                                                                      * , host
+                                                                      */)
+                                                        + 1 + 1 /*
+                                                                 * type,
+                                                                 * explicit_def
+                                                                 * ..ts
+                                                                 */+ 1 + 8 /*
+                                                                            * type,
+                                                                            * xid
+                                                                            * of
+                                                                            * DDL
+                                                                            */+ 1 + 2 /*
+                                                                                       * type
+                                                                                       * ,
+                                                                                       * default_collation_for_utf8mb4_number
+                                                                                       */+ 1 /* sql_require_primary_key */;
     /**
      * Fixed data part:
      * <ul>
@@ -394,7 +413,7 @@ public class QueryLogEvent extends LogEvent {
     // inspection by the DBA
     private final long      execTime;
     private final int       errorCode;
-    private final long      sessionId;                                                            /* thread_id */
+    private final long      sessionId;                                                                                     /* thread_id */
 
     /**
      * 'flags2' is a second set of flags (on top of those in Log_event), for
@@ -412,6 +431,8 @@ public class QueryLogEvent extends LogEvent {
     private int             clientCharset             = -1;
     private int             clientCollation           = -1;
     private int             serverCollation           = -1;
+    private int             tvSec                     = -1;
+    private BigInteger      ddlXid                    = BigInteger.valueOf(-1L);
     private String          charsetName;
 
     private String          timezone;
@@ -562,6 +583,21 @@ public class QueryLogEvent extends LogEvent {
      */
     public static final int Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP = 16;
 
+    /**
+     * The variable carries xid info of 2pc-aware (recoverable) DDL queries.
+     */
+    public static final int Q_DDL_LOGGED_WITH_XID             = 17;
+    /**
+     * This variable stores the default collation for the utf8mb4 character set.
+     * Used to support cross-version replication.
+     */
+    public static final int Q_DEFAULT_COLLATION_FOR_UTF8MB4   = 18;
+
+    /**
+     * Replicate sql_require_primary_key.
+     */
+    public static final int Q_SQL_REQUIRE_PRIMARY_KEY         = 19;
+
     /**
      * FROM MariaDB 5.5.34
      */
@@ -625,7 +661,7 @@ public class QueryLogEvent extends LogEvent {
                         break;
                     case Q_MICROSECONDS:
                         // when.tv_usec= uint3korr(pos);
-                        buffer.forward(3);
+                        tvSec = buffer.getInt24();
                         break;
                     case Q_UPDATED_DB_NAMES:
                         int mtsAccessedDbs = buffer.getUint8();
@@ -646,6 +682,19 @@ public class QueryLogEvent extends LogEvent {
                         }
                         break;
                     case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
+                        // thd->variables.explicit_defaults_for_timestamp
+                        buffer.forward(1);
+                        break;
+                    case Q_DDL_LOGGED_WITH_XID:
+                        ddlXid = buffer.getUlong64();
+                        break;
+                    case Q_DEFAULT_COLLATION_FOR_UTF8MB4:
+                        // int2store(start,
+                        // default_collation_for_utf8mb4_number);
+                        buffer.forward(2);
+                        break;
+                    case Q_SQL_REQUIRE_PRIMARY_KEY:
+                        // *start++ = thd->variables.sql_require_primary_key;
                         buffer.forward(1);
                         break;
                     case Q_HRNOW:
@@ -657,8 +706,10 @@ public class QueryLogEvent extends LogEvent {
                          * That's why you must write status vars in growing
                          * order of code
                          */
-                        if (logger.isDebugEnabled()) logger.debug("Query_log_event has unknown status vars (first has code: "
+                        if (logger.isDebugEnabled()) {
-                                                                  + code + "), skipping the rest of them");
+                            logger.debug("Query_log_event has unknown status vars (first has code: " + code
+                                         + "), skipping the rest of them");
+                        }
                         break; // Break loop
                 }
             }
@@ -695,6 +746,12 @@ public class QueryLogEvent extends LogEvent {
                 return "Q_UPDATED_DB_NAMES";
             case Q_MICROSECONDS:
                 return "Q_MICROSECONDS";
+            case Q_DDL_LOGGED_WITH_XID:
+                return "Q_DDL_LOGGED_WITH_XID";
+            case Q_DEFAULT_COLLATION_FOR_UTF8MB4:
+                return "Q_DEFAULT_COLLATION_FOR_UTF8MB4";
+            case Q_SQL_REQUIRE_PRIMARY_KEY:
+                return "Q_SQL_REQUIRE_PRIMARY_KEY";
         }
         return "CODE#" + code;
     }
@@ -777,6 +834,14 @@ public class QueryLogEvent extends LogEvent {
         return serverCollation;
     }
 
+    public int getTvSec() {
+        return tvSec;
+    }
+
+    public BigInteger getDdlXid() {
+        return ddlXid;
+    }
+
     /**
      * Returns the sql_mode value.
      * <p>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java

@@ -10,6 +10,7 @@ import org.apache.commons.logging.LogFactory;
 
 import com.taobao.tddl.dbsync.binlog.JsonConversion;
 import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
+import com.taobao.tddl.dbsync.binlog.JsonDiffConversion;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
@@ -31,22 +32,33 @@ public final class RowsLogBuffer {
 
     private final LogBuffer    buffer;
     private final int          columnLen;
+    private final int          jsonColumnCount;
     private final String       charsetName;
-    // private Calendar cal;
 
     private final BitSet       nullBits;
     private int                nullBitIndex;
 
+    // Read value_options if this is AI for PARTIAL_UPDATE_ROWS_EVENT
+    private final boolean      partial;
+    private final BitSet       partialBits;
+
     private boolean            fNull;
     private int                javaType;
     private int                length;
     private Serializable       value;
 
-    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName){
+    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName, int jsonColumnCount, boolean partial){
         this.buffer = buffer;
         this.columnLen = columnLen;
         this.charsetName = charsetName;
+        this.partial = partial;
+        this.jsonColumnCount = jsonColumnCount;
         this.nullBits = new BitSet(columnLen);
+        this.partialBits = new BitSet(1);
+    }
+
+    public final boolean nextOneRow(BitSet columns) {
+        return nextOneRow(columns, false);
     }
 
     /**
@@ -55,18 +67,30 @@ public final class RowsLogBuffer {
      * @see mysql-5.1.60/sql/log_event.cc -
      * Rows_log_event::print_verbose_one_row
      */
-    public final boolean nextOneRow(BitSet columns) {
+    public final boolean nextOneRow(BitSet columns, boolean after) {
         final boolean hasOneRow = buffer.hasRemaining();
 
         if (hasOneRow) {
             int column = 0;
 
             for (int i = 0; i < columnLen; i++)
-                if (columns.get(i)) column++;
+                if (columns.get(i)) {
+                    column++;
+                }
 
+            if (after && partial) {
+                partialBits.clear();
+                long valueOptions = buffer.getPackedLong();
+                int PARTIAL_JSON_UPDATES = 1;
+                if ((valueOptions & PARTIAL_JSON_UPDATES) != 0) {
+                    partialBits.set(1);
+                    buffer.forward((jsonColumnCount + 7) / 8);
+                }
+            }
             nullBitIndex = 0;
             nullBits.clear();
             buffer.fillBitmap(nullBits, column);
+
         }
         return hasOneRow;
     }
@@ -77,8 +101,8 @@ public final class RowsLogBuffer {
      * @see mysql-5.1.60/sql/log_event.cc -
      * Rows_log_event::print_verbose_one_row
      */
-    public final Serializable nextValue(final int type, final int meta) {
+    public final Serializable nextValue(final String columName, final int columnIndex, final int type, final int meta) {
-        return nextValue(type, meta, false);
+        return nextValue(columName, columnIndex, type, meta, false);
     }
 
     /**
@@ -87,7 +111,8 @@ public final class RowsLogBuffer {
      * @see mysql-5.1.60/sql/log_event.cc -
      * Rows_log_event::print_verbose_one_row
      */
-    public final Serializable nextValue(final int type, final int meta, boolean isBinary) {
+    public final Serializable nextValue(final String columName, final int columnIndex, final int type, final int meta,
+                                        boolean isBinary) {
         fNull = nullBits.get(nullBitIndex++);
 
         if (fNull) {
@@ -97,7 +122,7 @@ public final class RowsLogBuffer {
             return null;
         } else {
             // Extracting field value from packed buffer.
-            return fetchValue(type, meta, isBinary);
+            return fetchValue(columName, columnIndex, type, meta, isBinary);
         }
     }
 
@@ -248,7 +273,7 @@ public final class RowsLogBuffer {
      * 
      * @see mysql-5.1.60/sql/log_event.cc - log_event_print_value
      */
-    final Serializable fetchValue(int type, final int meta, boolean isBinary) {
+    final Serializable fetchValue(String columnName, int columnIndex, int type, final int meta, boolean isBinary) {
         int len = 0;
 
         if (type == LogEvent.MYSQL_TYPE_STRING) {
@@ -610,7 +635,7 @@ public final class RowsLogBuffer {
                     // (u32 % 10000) / 100,
                     // u32 % 100);
 
-                    StringBuilder builder = new StringBuilder(12);
+                    StringBuilder builder = new StringBuilder(17);
                     if (i32 < 0) {
                         builder.append('-');
                     }
@@ -1041,17 +1066,34 @@ public final class RowsLogBuffer {
                     default:
                         throw new IllegalArgumentException("!! Unknown JSON packlen = " + meta);
                 }
-                if (0 == len) {
+
-                    // fixed issue #1 by lava, json column of zero length has no
+                if (partialBits.get(1)) {
-                    // value, value parsing should be skipped
+                    // print_json_diff
-                    value = "";
-                } else {
                     int position = buffer.position();
-                    Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName);
+                    StringBuilder builder = JsonDiffConversion.print_json_diff(buffer,
-                    StringBuilder builder = new StringBuilder();
+                        len,
-                    jsonValue.toJsonString(builder, charsetName);
+                        columnName,
+                        columnIndex,
+                        charsetName);
                     value = builder.toString();
                     buffer.position(position + len);
+                } else {
+                    if (0 == len) {
+                        // fixed issue #1 by lava, json column of zero length
+                        // has no
+                        // value, value parsing should be skipped
+                        value = "";
+                    } else {
+                        int position = buffer.position();
+                        Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
+                            buffer,
+                            len - 1,
+                            charsetName);
+                        StringBuilder builder = new StringBuilder();
+                        jsonValue.toJsonString(builder, charsetName);
+                        value = builder.toString();
+                        buffer.position(position + len);
+                    }
                 }
                 javaType = Types.VARCHAR;
                 length = len;
@@ -1120,7 +1162,7 @@ public final class RowsLogBuffer {
         return length;
     }
 
-    private String usecondsToStr(int frac, int meta) {
+    public static String usecondsToStr(int frac, int meta) {
         String sec = String.valueOf(frac);
         if (meta > 6) {
             throw new IllegalArgumentException("unknow useconds meta : " + meta);
@@ -1139,7 +1181,7 @@ public final class RowsLogBuffer {
         return sec.substring(0, meta);
     }
 
-    private void appendNumber4(StringBuilder builder, int d) {
+    public static void appendNumber4(StringBuilder builder, int d) {
         if (d >= 1000) {
             builder.append(digits[d / 1000])
                 .append(digits[(d / 100) % 10])
@@ -1151,7 +1193,7 @@ public final class RowsLogBuffer {
         }
     }
 
-    private void appendNumber3(StringBuilder builder, int d) {
+    public static void appendNumber3(StringBuilder builder, int d) {
         if (d >= 100) {
             builder.append(digits[d / 100]).append(digits[(d / 10) % 10]).append(digits[d % 10]);
         } else {
@@ -1160,7 +1202,7 @@ public final class RowsLogBuffer {
         }
     }
 
-    private void appendNumber2(StringBuilder builder, int d) {
+    public static void appendNumber2(StringBuilder builder, int d) {
         if (d >= 10) {
             builder.append(digits[(d / 10) % 10]).append(digits[d % 10]);
         } else {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java

@@ -5,6 +5,7 @@ import java.util.BitSet;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
 
 /**
  * Common base class for all row-containing log events.
@@ -61,6 +62,7 @@ public abstract class RowsLogEvent extends LogEvent {
 
     /** Bitmap denoting columns available */
     protected final int      columnLen;
+    protected final boolean  partial;
     protected final BitSet   columns;
 
     /**
@@ -71,6 +73,8 @@ public abstract class RowsLogEvent extends LogEvent {
      */
     protected final BitSet   changeColumns;
 
+    protected int            jsonColumnCount         = 0;
+
     /** XXX: Don't handle buffer in another thread. */
     private final LogBuffer  rowsBuf;                           /*
                                                                   * The rows in
@@ -109,6 +113,10 @@ public abstract class RowsLogEvent extends LogEvent {
     public static final int  RW_V_EXTRAINFO_TAG      = 0;
 
     public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        this(header, buffer, descriptionEvent, false);
+    }
+
+    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, boolean partial){
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -153,9 +161,11 @@ public abstract class RowsLogEvent extends LogEvent {
 
         buffer.position(commonHeaderLen + postHeaderLen + headerLen);
         columnLen = (int) buffer.getPackedLong();
+        this.partial = partial;
         columns = buffer.getBitmap(columnLen);
 
-        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) {
+        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT
+            || header.type == PARTIAL_UPDATE_ROWS_EVENT) {
             changeColumns = buffer.getBitmap(columnLen);
         } else {
             changeColumns = columns;
@@ -175,6 +185,17 @@ public abstract class RowsLogEvent extends LogEvent {
             // delete original table map events stored in the map).
             context.clearAllTables();
         }
+
+        int jsonColumnCount = 0;
+        int columnCnt = table.getColumnCnt();
+        ColumnInfo[] columnInfo = table.getColumnInfo();
+        for (int i = 0; i < columnCnt; i++) {
+            ColumnInfo info = columnInfo[i];
+            if (info.type == LogEvent.MYSQL_TYPE_JSON) {
+                jsonColumnCount++;
+            }
+        }
+        this.jsonColumnCount = jsonColumnCount;
     }
 
     public final long getTableId() {
@@ -194,7 +215,7 @@ public abstract class RowsLogEvent extends LogEvent {
     }
 
     public final RowsLogBuffer getRowsBuf(String charsetName) {
-        return new RowsLogBuffer(rowsBuf, columnLen, charsetName);
+        return new RowsLogBuffer(rowsBuf, columnLen, charsetName, jsonColumnCount, partial);
     }
 
     public final int getFlags(final int flags) {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java

@@ -1,6 +1,8 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
+import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.List;
 
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
@@ -88,6 +90,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * first byte, the ninth is in the least significant bit of the second byte, and
  * so on.</td>
  * </tr>
+ * <tr>
+ * <td>optional metadata fields</td>
+ * <td>optional metadata fields are stored in Type, Length, Value(TLV) format.
+ * Type takes 1 byte. Length is a packed integer value. Values takes Length
+ * bytes.</td>
+ * <td>There are some optional metadata defined. They are listed in the table
+ * 
+ * @ref Table_table_map_event_optional_metadata. Optional metadata fields follow
+ * null_bits. Whether binlogging an optional metadata is decided by the server.
+ * The order is not defined, so they can be binlogged in any order. </td>
+ * </tr>
  * </table>
  * The table below lists all column types, along with the numerical identifier
  * for it and the size and interpretation of meta-data used to describe the
@@ -284,7 +297,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * of the geometry: 1, 2, 3, or 4.</td>
  * </tr>
  * </table>
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
@@ -322,20 +334,63 @@ public final class TableMapLogEvent extends LogEvent {
      */
     public static final class ColumnInfo {
 
-        public int type;
+        public int          type;
-        public int meta;
+        public int          meta;
+        public String       name;
+        public boolean      unsigned;
+        public boolean      pk;
+        public List<String> set_enum_values;
+        public int          charset;        // 可以通过CharsetUtil进行转化
+        public int          geoType;
+        public boolean      nullable;
+
+        @Override
+        public String toString() {
+            return "ColumnInfo [type=" + type + ", meta=" + meta + ", name=" + name + ", unsigned=" + unsigned
+                   + ", pk=" + pk + ", set_enum_values=" + set_enum_values + ", charset=" + charset + ", geoType="
+                   + geoType + ", nullable=" + nullable + "]";
+        }
     }
 
     protected final int          columnCnt;
-    protected final ColumnInfo[] columnInfo;         // buffer for field
+    protected final ColumnInfo[] columnInfo;                     // buffer
-                                                      // metadata
+                                                                  // for
+                                                                  // field
+                                                                  // metadata
 
     protected final long         tableId;
     protected BitSet             nullBits;
 
     /** TM = "Table Map" */
-    public static final int      TM_MAPID_OFFSET = 0;
+    public static final int      TM_MAPID_OFFSET         = 0;
-    public static final int      TM_FLAGS_OFFSET = 6;
+    public static final int      TM_FLAGS_OFFSET         = 6;
+
+    // UNSIGNED flag of numeric columns
+    public static final int      SIGNEDNESS              = 1;
+    // Default character set of string columns
+    public static final int      DEFAULT_CHARSET         = 2;
+    // Character set of string columns
+    public static final int      COLUMN_CHARSET          = 3;
+    public static final int      COLUMN_NAME             = 4;
+    // String value of SET columns
+    public static final int      SET_STR_VALUE           = 5;
+    // String value of ENUM columns
+    public static final int      ENUM_STR_VALUE          = 6;
+    // Real type of geometry columns
+    public static final int      GEOMETRY_TYPE           = 7;
+    // Primary key without prefix
+    public static final int      SIMPLE_PRIMARY_KEY      = 8;
+    // Primary key with prefix
+    public static final int      PRIMARY_KEY_WITH_PREFIX = 9;
+
+    private int                  default_charset;
+    private boolean              existOptionalMetaData   = false;
+
+    private static final class Pair {
+
+        public int col_index;
+        public int col_charset;
+    }
 
     /**
      * Constructor used by slave to read the event from the binary log.
@@ -379,7 +434,91 @@ public final class TableMapLogEvent extends LogEvent {
             final int fieldSize = (int) buffer.getPackedLong();
             decodeFields(buffer, fieldSize);
             nullBits = buffer.getBitmap(columnCnt);
+
+            for (int i = 0; i < columnCnt; i++) {
+                if (nullBits.get(i)) {
+                    columnInfo[i].nullable = true;
+                }
+            }
+            /*
+             * After null_bits field, there are some new fields for extra
+             * metadata.
+             */
+            existOptionalMetaData = false;
+            List<TableMapLogEvent.Pair> defaultCharsetPairs = null;
+            List<Integer> columnCharsets = null;
+            while (buffer.hasRemaining()) {
+                // optional metadata fields
+                int type = buffer.getUint8();
+                int len = (int) buffer.getPackedLong();
+
+                switch (type) {
+                    case SIGNEDNESS:
+                        parse_signedness(buffer, len);
+                        break;
+                    case DEFAULT_CHARSET:
+                        defaultCharsetPairs = parse_default_charset(buffer, len);
+                        break;
+                    case COLUMN_CHARSET:
+                        columnCharsets = parse_column_charset(buffer, len);
+                        break;
+                    case COLUMN_NAME:
+                        // set @@global.binlog_row_metadata='FULL'
+                        // 主要是补充列名相关信息
+                        existOptionalMetaData = true;
+                        parse_column_name(buffer, len);
+                        break;
+                    case SET_STR_VALUE:
+                        parse_set_str_value(buffer, len, true);
+                        break;
+                    case ENUM_STR_VALUE:
+                        parse_set_str_value(buffer, len, false);
+                        break;
+                    case GEOMETRY_TYPE:
+                        parse_geometry_type(buffer, len);
+                        break;
+                    case SIMPLE_PRIMARY_KEY:
+                        parse_simple_pk(buffer, len);
+                        break;
+                    case PRIMARY_KEY_WITH_PREFIX:
+                        parse_pk_with_prefix(buffer, len);
+                        break;
+                    default:
+                        throw new IllegalArgumentException("unknow type : " + type);
+                }
+            }
+
+            if (existOptionalMetaData) {
+                int index = 0;
+                int char_col_index = 0;
+                for (int i = 0; i < columnCnt; i++) {
+                    int cs = -1;
+                    int type = getRealType(columnInfo[i].type, columnInfo[i].meta);
+                    if (is_character_type(type)) {
+                        if (defaultCharsetPairs != null && !defaultCharsetPairs.isEmpty()) {
+                            if (index < defaultCharsetPairs.size()
+                                && char_col_index == defaultCharsetPairs.get(index).col_index) {
+                                cs = defaultCharsetPairs.get(index).col_charset;
+                                index++;
+                            } else {
+                                cs = default_charset;
+                            }
+
+                            char_col_index++;
+                        } else if (columnCharsets != null) {
+                            cs = columnCharsets.get(index);
+                            index++;
+                        }
+
+                        columnInfo[i].charset = cs;
+                    }
+                }
+            }
         }
+
+        // for (int i = 0; i < columnCnt; i++) {
+        // System.out.println(columnInfo[i]);
+        // }
     }
 
     /**
@@ -459,6 +598,192 @@ public final class TableMapLogEvent extends LogEvent {
         buffer.limit(limit);
     }
 
+    private void parse_signedness(LogBuffer buffer, int length) {
+        // stores the signedness flags extracted from field
+        List<Boolean> datas = new ArrayList<Boolean>();
+        for (int i = 0; i < length; i++) {
+            int ut = buffer.getUint8();
+            for (int c = 0x80; c != 0; c >>= 1) {
+                datas.add((ut & c) > 0);
+            }
+        }
+
+        int index = 0;
+        for (int i = 0; i < columnCnt; i++) {
+            if (is_numeric_type(columnInfo[i].type)) {
+                columnInfo[i].unsigned = datas.get(index);
+                index++;
+            }
+        }
+    }
+
+    private List<TableMapLogEvent.Pair> parse_default_charset(LogBuffer buffer, int length) {
+        // stores collation numbers extracted from field.
+        int limit = buffer.position() + length;
+        this.default_charset = (int) buffer.getPackedLong();
+        List<TableMapLogEvent.Pair> datas = new ArrayList<TableMapLogEvent.Pair>();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_index = (int) buffer.getPackedLong();
+            int col_charset = (int) buffer.getPackedLong();
+
+            Pair pair = new Pair();
+            pair.col_index = col_index;
+            pair.col_charset = col_charset;
+            datas.add(pair);
+        }
+
+        return datas;
+    }
+
+    private List<Integer> parse_column_charset(LogBuffer buffer, int length) {
+        // stores collation numbers extracted from field.
+        int limit = buffer.position() + length;
+        List<Integer> datas = new ArrayList<Integer>();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_charset = (int) buffer.getPackedLong();
+            datas.add(col_charset);
+        }
+
+        return datas;
+    }
+
+    private void parse_column_name(LogBuffer buffer, int length) {
+        // stores column names extracted from field
+        int limit = buffer.position() + length;
+        int index = 0;
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int len = (int) buffer.getPackedLong();
+            columnInfo[index++].name = buffer.getFixString(len);
+        }
+    }
+
+    private void parse_set_str_value(LogBuffer buffer, int length, boolean set) {
+        // stores SET/ENUM column's string values extracted from
+        // field. Each SET/ENUM column's string values are stored
+        // into a string separate vector. All of them are stored
+        // in 'vec'.
+        int limit = buffer.position() + length;
+        List<List<String>> datas = new ArrayList<List<String>>();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int count = (int) buffer.getPackedLong();
+            List<String> data = new ArrayList<String>(count);
+            for (int i = 0; i < count; i++) {
+                int len1 = (int) buffer.getPackedLong();
+                data.add(buffer.getFixString(len1));
+            }
+
+            datas.add(data);
+        }
+
+        int index = 0;
+        for (int i = 0; i < columnCnt; i++) {
+            if (set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_SET) {
+                columnInfo[i].set_enum_values = datas.get(index);
+                index++;
+            }
+
+            if (!set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_ENUM) {
+                columnInfo[i].set_enum_values = datas.get(index);
+                index++;
+            }
+        }
+    }
+
+    private void parse_geometry_type(LogBuffer buffer, int length) {
+        // stores geometry column's types extracted from field.
+        int limit = buffer.position() + length;
+
+        List<Integer> datas = new ArrayList<Integer>();
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_type = (int) buffer.getPackedLong();
+            datas.add(col_type);
+        }
+
+        int index = 0;
+        for (int i = 0; i < columnCnt; i++) {
+            if (columnInfo[i].type == LogEvent.MYSQL_TYPE_GEOMETRY) {
+                columnInfo[i].geoType = datas.get(index);
+                index++;
+            }
+        }
+    }
+
+    private void parse_simple_pk(LogBuffer buffer, int length) {
+        // stores primary key's column information extracted from
+        // field. Each column has an index and a prefix which are
+        // stored as a unit_pair. prefix is always 0 for
+        // SIMPLE_PRIMARY_KEY field.
+
+        int limit = buffer.position() + length;
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_index = (int) buffer.getPackedLong();
+            columnInfo[col_index].pk = true;
+        }
+    }
+
+    private void parse_pk_with_prefix(LogBuffer buffer, int length) {
+        // stores primary key's column information extracted from
+        // field. Each column has an index and a prefix which are
+        // stored as a unit_pair.
+        int limit = buffer.position() + length;
+        while (buffer.hasRemaining() && buffer.position() < limit) {
+            int col_index = (int) buffer.getPackedLong();
+            // prefix length, 比如 char(32)
+            @SuppressWarnings("unused")
+            int col_prefix = (int) buffer.getPackedLong();
+            columnInfo[col_index].pk = true;
+        }
+    }
+
+    private boolean is_numeric_type(int type) {
+        switch (type) {
+            case MYSQL_TYPE_TINY:
+            case MYSQL_TYPE_SHORT:
+            case MYSQL_TYPE_INT24:
+            case MYSQL_TYPE_LONG:
+            case MYSQL_TYPE_LONGLONG:
+            case MYSQL_TYPE_NEWDECIMAL:
+            case MYSQL_TYPE_FLOAT:
+            case MYSQL_TYPE_DOUBLE:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    private boolean is_character_type(int type) {
+        switch (type) {
+            case MYSQL_TYPE_STRING:
+            case MYSQL_TYPE_VAR_STRING:
+            case MYSQL_TYPE_VARCHAR:
+            case MYSQL_TYPE_BLOB:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    private int getRealType(int type, int meta) {
+        if (type == LogEvent.MYSQL_TYPE_STRING) {
+            if (meta >= 256) {
+                int byte0 = meta >> 8;
+                if ((byte0 & 0x30) != 0x30) {
+                    /* a long CHAR() field: see #37426 */
+                    type = byte0 | 0x30;
+                } else {
+                    switch (byte0) {
+                        case LogEvent.MYSQL_TYPE_SET:
+                        case LogEvent.MYSQL_TYPE_ENUM:
+                        case LogEvent.MYSQL_TYPE_STRING:
+                            type = byte0;
+                    }
+                }
+            }
+        }
+
+        return type;
+    }
+
     public final String getDbName() {
         return dbname;
     }
@@ -478,4 +803,13 @@ public final class TableMapLogEvent extends LogEvent {
     public final long getTableId() {
         return tableId;
     }
+
+    public boolean isExistOptionalMetaData() {
+        return existOptionalMetaData;
+    }
+
+    public void setExistOptionalMetaData(boolean existOptional) {
+        this.existOptionalMetaData = existOptional;
+    }
+
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java

@@ -14,6 +14,11 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 public final class UpdateRowsLogEvent extends RowsLogEvent {
 
     public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-        super(header, buffer, descriptionEvent);
+        super(header, buffer, descriptionEvent, false);
+    }
+
+    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                              boolean partial){
+        super(header, buffer, descriptionEvent, partial);
     }
 }

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java

@@ -68,7 +68,7 @@ public class BaseLogFetcherTest {
                     // update需要处理before/after
                     System.out.println("-------> before");
                     parseOneRow(event, buffer, columns, false);
-                    if (!buffer.nextOneRow(changeColumns)) {
+                    if (!buffer.nextOneRow(changeColumns, true)) {
                         break;
                     }
                     System.out.println("-------> after");
@@ -97,7 +97,7 @@ public class BaseLogFetcherTest {
             }
 
             ColumnInfo info = columnInfo[i];
-            buffer.nextValue(info.type, info.meta);
+            buffer.nextValue(null , i ,info.type, info.meta);
 
             if (buffer.isNull()) {
                 //

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java

@@ -45,6 +45,7 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
                         parseRowsEvent((WriteRowsLogEvent) event);
                         break;
                     case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                     case LogEvent.UPDATE_ROWS_EVENT:
                         parseRowsEvent((UpdateRowsLogEvent) event);
                         break;

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java

@@ -53,6 +53,7 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
                             parseRowsEvent((WriteRowsLogEvent) event);
                             break;
                         case LogEvent.UPDATE_ROWS_EVENT_V1:
+                        case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                         case LogEvent.UPDATE_ROWS_EVENT:
                             parseRowsEvent((UpdateRowsLogEvent) event);
                             break;

deployer/src/main/resources/mq.yml

@@ -1,4 +1,4 @@
-servers: localhost:9876 #for rocketmq: means the nameserver
+servers: slave1:6667 #for rocketmq: means the nameserver
 retries: 0
 batchSize: 16384
 lingerMs: 1
@@ -13,7 +13,7 @@ flatMessage: true
 canalDestinations:
   - canalDestination: example
     topic: example
-    partition: 1
+    partition:
 #  #对应topic分区数量
 #  partitionsNum: 3
 #  partitionHash:

deployer/src/main/resources/spring/tsdb/h2-tsdb.xml

@@ -23,7 +23,7 @@
 	</bean>
 	
 	<!-- 基于db的实现 -->
-	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
 		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
 		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
 	</bean>

deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml

@@ -23,7 +23,7 @@
 	</bean>
 	
 	<!-- 基于db的实现 -->
-	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
 		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
 		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
 	</bean>

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java

@@ -88,8 +88,9 @@ public class BioSocketChannel implements SocketChannel {
             }
         }
         if (remain > 0 && accTimeout >= timeout) {
-            throw new SocketTimeoutException("Timeout occurred, failed to read " + readSize + " bytes in " + timeout
+            throw new SocketTimeoutException("Timeout occurred, failed to read total " + readSize + " bytes in "
-                                             + " milliseconds.");
+                                             + timeout + " milliseconds, actual read only " + (readSize - remain)
+                                             + " bytes");
         }
         return data;
     }
@@ -120,8 +121,8 @@ public class BioSocketChannel implements SocketChannel {
         }
 
         if (n < len && accTimeout >= timeout) {
-            throw new SocketTimeoutException("Timeout occurred, failed to read " + len + " bytes in " + timeout
+            throw new SocketTimeoutException("Timeout occurred, failed to read total " + len + " bytes in " + timeout
-                                             + " milliseconds.");
+                                             + " milliseconds, actual read only " + n + " bytes");
         }
     }
 

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java

@@ -0,0 +1,258 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * mysql collation转换mapping关系表
+ * 
+ * @author agapple 2018年11月5日 下午1:01:15
+ * @since 1.1.2
+ */
+public class CharsetUtil {
+
+    private static final String[]             INDEX_TO_CHARSET = new String[2048];
+    private static final Map<String, Integer> CHARSET_TO_INDEX = new HashMap<String, Integer>();
+    static {
+        INDEX_TO_CHARSET[1] = "big5";
+        INDEX_TO_CHARSET[84] = "big5";
+
+        INDEX_TO_CHARSET[3] = "dec8";
+        INDEX_TO_CHARSET[69] = "dec8";
+
+        INDEX_TO_CHARSET[4] = "cp850";
+        INDEX_TO_CHARSET[80] = "cp850";
+
+        INDEX_TO_CHARSET[6] = "hp8";
+        INDEX_TO_CHARSET[72] = "hp8";
+
+        INDEX_TO_CHARSET[7] = "koi8r";
+        INDEX_TO_CHARSET[74] = "koi8r";
+
+        INDEX_TO_CHARSET[5] = "latin1";
+        INDEX_TO_CHARSET[8] = "latin1";
+        INDEX_TO_CHARSET[15] = "latin1";
+        INDEX_TO_CHARSET[31] = "latin1";
+        INDEX_TO_CHARSET[47] = "latin1";
+        INDEX_TO_CHARSET[48] = "latin1";
+        INDEX_TO_CHARSET[49] = "latin1";
+        INDEX_TO_CHARSET[94] = "latin1";
+
+        INDEX_TO_CHARSET[9] = "latin2";
+        INDEX_TO_CHARSET[21] = "latin2";
+        INDEX_TO_CHARSET[27] = "latin2";
+        INDEX_TO_CHARSET[77] = "latin2";
+
+        INDEX_TO_CHARSET[10] = "swe7";
+        INDEX_TO_CHARSET[82] = "swe7";
+
+        INDEX_TO_CHARSET[11] = "ascii";
+        INDEX_TO_CHARSET[65] = "ascii";
+
+        INDEX_TO_CHARSET[12] = "ujis";
+        INDEX_TO_CHARSET[91] = "ujis";
+
+        INDEX_TO_CHARSET[13] = "sjis";
+        INDEX_TO_CHARSET[88] = "sjis";
+
+        INDEX_TO_CHARSET[16] = "hebrew";
+        INDEX_TO_CHARSET[71] = "hebrew";
+
+        INDEX_TO_CHARSET[18] = "tis620";
+        INDEX_TO_CHARSET[69] = "tis620";
+
+        INDEX_TO_CHARSET[19] = "euckr";
+        INDEX_TO_CHARSET[85] = "euckr";
+
+        INDEX_TO_CHARSET[22] = "koi8u";
+        INDEX_TO_CHARSET[75] = "koi8u";
+
+        INDEX_TO_CHARSET[24] = "gb2312";
+        INDEX_TO_CHARSET[86] = "gb2312";
+
+        INDEX_TO_CHARSET[25] = "greek";
+        INDEX_TO_CHARSET[70] = "greek";
+
+        INDEX_TO_CHARSET[26] = "cp1250";
+        INDEX_TO_CHARSET[34] = "cp1250";
+        INDEX_TO_CHARSET[44] = "cp1250";
+        INDEX_TO_CHARSET[66] = "cp1250";
+        INDEX_TO_CHARSET[99] = "cp1250";
+
+        INDEX_TO_CHARSET[28] = "gbk";
+        INDEX_TO_CHARSET[87] = "gbk";
+
+        INDEX_TO_CHARSET[30] = "latin5";
+        INDEX_TO_CHARSET[78] = "latin5";
+
+        INDEX_TO_CHARSET[32] = "armscii8";
+        INDEX_TO_CHARSET[64] = "armscii8";
+
+        INDEX_TO_CHARSET[33] = "utf8";
+        INDEX_TO_CHARSET[83] = "utf8";
+        for (int i = 192; i <= 223; i++) {
+            INDEX_TO_CHARSET[i] = "utf8";
+        }
+        for (int i = 336; i <= 337; i++) {
+            INDEX_TO_CHARSET[i] = "utf8";
+        }
+        for (int i = 352; i <= 357; i++) {
+            INDEX_TO_CHARSET[i] = "utf8";
+        }
+        INDEX_TO_CHARSET[368] = "utf8";
+        INDEX_TO_CHARSET[2047] = "utf8";
+
+        INDEX_TO_CHARSET[35] = "ucs2";
+        INDEX_TO_CHARSET[90] = "ucs2";
+        for (int i = 128; i <= 151; i++) {
+            INDEX_TO_CHARSET[i] = "ucs2";
+        }
+        INDEX_TO_CHARSET[159] = "ucs2";
+        for (int i = 358; i <= 360; i++) {
+            INDEX_TO_CHARSET[i] = "ucs2";
+        }
+
+        INDEX_TO_CHARSET[36] = "cp866";
+        INDEX_TO_CHARSET[68] = "cp866";
+
+        INDEX_TO_CHARSET[37] = "keybcs2";
+        INDEX_TO_CHARSET[73] = "keybcs2";
+
+        INDEX_TO_CHARSET[38] = "macce";
+        INDEX_TO_CHARSET[43] = "macce";
+
+        INDEX_TO_CHARSET[39] = "macroman";
+        INDEX_TO_CHARSET[53] = "macroman";
+
+        INDEX_TO_CHARSET[40] = "cp852";
+        INDEX_TO_CHARSET[81] = "cp852";
+
+        INDEX_TO_CHARSET[20] = "latin7";
+        INDEX_TO_CHARSET[41] = "latin7";
+        INDEX_TO_CHARSET[42] = "latin7";
+        INDEX_TO_CHARSET[79] = "latin7";
+
+        INDEX_TO_CHARSET[45] = "utf8mb4";
+        INDEX_TO_CHARSET[46] = "utf8mb4";
+        for (int i = 224; i <= 247; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 255; i <= 271; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 273; i <= 275; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 277; i <= 294; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        for (int i = 296; i <= 298; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        INDEX_TO_CHARSET[300] = "utf8mb4";
+        for (int i = 303; i <= 307; i++) {
+            INDEX_TO_CHARSET[i] = "utf8mb4";
+        }
+        INDEX_TO_CHARSET[326] = "utf8mb4";
+        INDEX_TO_CHARSET[328] = "utf8mb4";
+
+        INDEX_TO_CHARSET[14] = "cp1251";
+        INDEX_TO_CHARSET[23] = "cp1251";
+        INDEX_TO_CHARSET[50] = "cp1251";
+        INDEX_TO_CHARSET[51] = "cp1251";
+        INDEX_TO_CHARSET[52] = "cp1251";
+
+        INDEX_TO_CHARSET[54] = "utf16";
+        INDEX_TO_CHARSET[55] = "utf16";
+        for (int i = 101; i <= 124; i++) {
+            INDEX_TO_CHARSET[i] = "utf16";
+        }
+        INDEX_TO_CHARSET[327] = "utf16";
+
+        INDEX_TO_CHARSET[56] = "utf16le";
+        INDEX_TO_CHARSET[62] = "utf16le";
+
+        INDEX_TO_CHARSET[57] = "cp1256";
+        INDEX_TO_CHARSET[67] = "cp1256";
+
+        INDEX_TO_CHARSET[29] = "cp1257";
+        INDEX_TO_CHARSET[58] = "cp1257";
+        INDEX_TO_CHARSET[59] = "cp1257";
+
+        INDEX_TO_CHARSET[60] = "utf32";
+        INDEX_TO_CHARSET[61] = "utf32";
+        for (int i = 160; i <= 183; i++) {
+            INDEX_TO_CHARSET[i] = "utf32";
+        }
+        INDEX_TO_CHARSET[391] = "utf32";
+
+        INDEX_TO_CHARSET[63] = "binary";
+
+        INDEX_TO_CHARSET[92] = "geostd8";
+        INDEX_TO_CHARSET[93] = "geostd8";
+
+        INDEX_TO_CHARSET[95] = "cp932";
+        INDEX_TO_CHARSET[96] = "cp932";
+
+        INDEX_TO_CHARSET[97] = "eucjpms";
+        INDEX_TO_CHARSET[98] = "eucjpms";
+
+        for (int i = 248; i <= 250; i++) {
+            INDEX_TO_CHARSET[i] = "gb18030";
+        }
+
+        // charset --> index
+        for (int i = 0; i < 2048; i++) {
+            String charset = INDEX_TO_CHARSET[i];
+            if (charset != null && CHARSET_TO_INDEX.get(charset) == null) {
+                CHARSET_TO_INDEX.put(charset, i);
+            }
+        }
+        CHARSET_TO_INDEX.put("iso-8859-1", 14);
+        CHARSET_TO_INDEX.put("iso_8859_1", 14);
+        CHARSET_TO_INDEX.put("utf-8", 33);
+        CHARSET_TO_INDEX.put("utf8mb4", 45);
+    }
+
+    public static final String getCharset(int index) {
+        return INDEX_TO_CHARSET[index];
+    }
+
+    public static final int getIndex(String charset) {
+        if (charset == null || charset.length() == 0) {
+            return 0;
+        } else {
+            Integer i = CHARSET_TO_INDEX.get(charset.toLowerCase());
+            return (i == null) ? 0 : i.intValue();
+        }
+    }
+
+    /**
+     * 'utf8' COLLATE 'utf8_general_ci'
+     * 
+     * @param charset
+     * @return
+     */
+    public static final String collateCharset(String charset) {
+        String[] output = StringUtils.split(charset, "COLLATE");
+        return output[0].replace('\'', ' ').trim();
+    }
+
+    public static String getJavaCharset(String charset) {
+        if ("utf8".equals(charset)) {
+            return charset;
+        }
+
+        if (StringUtils.endsWithIgnoreCase(charset, "utf8mb4")) {
+            return "utf-8";
+        }
+
+        if (StringUtils.endsWithIgnoreCase(charset, "binary")) {
+            return "iso_8859_1";
+        }
+
+        return charset;
+    }
+}

driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/CharsetUtilTest.java

@@ -0,0 +1,51 @@
+package com.alibaba.otter.canal.parse.driver.mysql;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.driver.mysql.utils.CharsetUtil;
+
+public class CharsetUtilTest {
+
+    @Test
+    public void testLatin1() {
+        int charsetIndex = 5;
+        String charset = "latin1";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testGbk() {
+        int charsetIndex = 87;
+        String charset = "gbk";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testGb2312() {
+        int charsetIndex = 24;
+        String charset = "gb2312";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testUtf8() {
+        int charsetIndex = 213;
+        String charset = "utf8";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testUtf8mb4() {
+        int charsetIndex = 235;
+        String charset = "utf8mb4";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+
+    @Test
+    public void testBinary() {
+        int charsetIndex = 63;
+        String charset = "binary";
+        Assert.assertTrue(charset.equals(CharsetUtil.getCharset(charsetIndex)));
+    }
+}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java

@@ -353,7 +353,11 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
         eventSink.interrupt();
 
         if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
-            multiStageCoprocessor.stop();
+            try {
+                multiStageCoprocessor.stop();
+            } catch (Throwable t) {
+                logger.debug("multi processor rejected:", t);
+            }
         }
 
         try {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java

@@ -60,6 +60,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         convert.setFilterQueryDdl(filterQueryDdl);
         convert.setFilterRows(filterRows);
         convert.setFilterTableError(filterTableError);
+        convert.setUseDruidDdlFilter(useDruidDdlFilter);
         return convert;
     }
 

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java

@@ -37,21 +37,23 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 
 public class MysqlConnection implements ErosaConnection {
 
-    private static final Logger logger      = LoggerFactory.getLogger(MysqlConnection.class);
+    private static final Logger logger         = LoggerFactory.getLogger(MysqlConnection.class);
 
     private MysqlConnector      connector;
     private long                slaveId;
-    private Charset             charset     = Charset.forName("UTF-8");
+    private Charset             charset        = Charset.forName("UTF-8");
     private BinlogFormat        binlogFormat;
     private BinlogImage         binlogImage;
 
     // tsdb releated
     private AuthenticationInfo  authInfo;
-    protected int               connTimeout = 5 * 1000;                                      // 5秒
+    protected int               connTimeout    = 5 * 1000;                                      // 5秒
-    protected int               soTimeout   = 60 * 60 * 1000;                                // 1小时
+    protected int               soTimeout      = 60 * 60 * 1000;                                // 1小时
+    private int                 binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
     // dump binlog bytes, 暂不包括meta与TSDB
     private AtomicLong          receivedBinlogBytes;
 
@@ -118,7 +120,7 @@ public class MysqlConnection implements ErosaConnection {
      */
     public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
         updateSettings();
-
+        loadBinlogChecksum();
         sendBinlogDump(binlogfilename, binlogPosition);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         fetcher.start(connector.getChannel());
@@ -128,6 +130,7 @@ public class MysqlConnection implements ErosaConnection {
         decoder.handle(LogEvent.QUERY_EVENT);
         decoder.handle(LogEvent.XID_EVENT);
         LogContext context = new LogContext();
+        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
         while (fetcher.fetch()) {
             accumulateReceivedBytes(fetcher.limit());
             LogEvent event = null;
@@ -145,12 +148,14 @@ public class MysqlConnection implements ErosaConnection {
 
     public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendRegisterSlave();
         sendBinlogDump(binlogfilename, binlogPosition);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         fetcher.start(connector.getChannel());
         LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
         LogContext context = new LogContext();
+        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
         while (fetcher.fetch()) {
             accumulateReceivedBytes(fetcher.limit());
             LogEvent event = null;
@@ -173,6 +178,7 @@ public class MysqlConnection implements ErosaConnection {
     @Override
     public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendBinlogDumpGTID(gtidSet);
 
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
@@ -180,6 +186,7 @@ public class MysqlConnection implements ErosaConnection {
             fetcher.start(connector.getChannel());
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
+            context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
             // fix bug: #890 将gtid传输至context中，供decode使用
             context.setGtidSet(gtidSet);
             while (fetcher.fetch()) {
@@ -207,9 +214,11 @@ public class MysqlConnection implements ErosaConnection {
     @Override
     public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendRegisterSlave();
         sendBinlogDump(binlogfilename, binlogPosition);
         ((MysqlMultiStageCoprocessor) coprocessor).setConnection(this);
+        ((MysqlMultiStageCoprocessor) coprocessor).setBinlogChecksum(binlogChecksum);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         try {
             fetcher.start(connector.getChannel());
@@ -234,9 +243,10 @@ public class MysqlConnection implements ErosaConnection {
     @Override
     public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException {
         updateSettings();
+        loadBinlogChecksum();
         sendBinlogDumpGTID(gtidSet);
-
         ((MysqlMultiStageCoprocessor) coprocessor).setConnection(this);
+        ((MysqlMultiStageCoprocessor) coprocessor).setBinlogChecksum(binlogChecksum);
         DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
         try {
             fetcher.start(connector.getChannel());
@@ -482,6 +492,52 @@ public class MysqlConnection implements ErosaConnection {
         }
     }
 
+    /**
+     * 获取主库checksum信息
+     * 
+     * <pre>
+     * mariadb区别于mysql会在binlog的第一个事件Rotate_Event里也会采用checksum逻辑,而mysql是在第二个binlog事件之后才感知是否需要处理checksum
+     * 导致maraidb只要是开启checksum就会出现binlog文件名解析乱码
+     * fixed issue : https://github.com/alibaba/canal/issues/1081
+     * </pre>
+     */
+    private void loadBinlogChecksum() {
+        if (checkMariaDB()) {
+            ResultSetPacket rs = null;
+            try {
+                rs = query("select @@global.binlog_checksum");
+            } catch (IOException e) {
+                throw new CanalParseException(e);
+            }
+
+            List<String> columnValues = rs.getFieldValues();
+            if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
+                binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
+            } else {
+                binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
+            }
+        }
+    }
+
+    /**
+     * 获取是否为mariadb
+     */
+    private boolean checkMariaDB() {
+        ResultSetPacket rs = null;
+        try {
+            rs = query("SELECT @@version");
+        } catch (IOException e) {
+            throw new CanalParseException(e);
+        }
+
+        List<String> columnValues = rs.getFieldValues();
+        if (columnValues != null && columnValues.size() >= 1) {
+            return StringUtils.containsIgnoreCase(columnValues.get(0), "MariaDB");
+        }
+
+        return false;
+    }
+
     private void accumulateReceivedBytes(long x) {
         if (receivedBinlogBytes != null) {
             receivedBinlogBytes.addAndGet(x);

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java

@@ -69,6 +69,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
     // update by yishun.chen,特殊异常处理参数
     private int                  dumpErrorCount                    = 0;        // binlogDump失败异常计数
     private int                  dumpErrorCountThreshold           = 2;        // binlogDump失败异常计数阀值
+    private boolean              rdsOssMode                        = false;
 
     protected ErosaConnection buildErosaConnection() {
         return buildMysqlConnection(this.runningInfo);
@@ -352,7 +353,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                 return logPosition.getPostion();
             }
 
-            if (masterPosition!=null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
+            if (masterPosition != null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
                 return masterPosition;
             }
         }
@@ -493,6 +494,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                         dumpErrorCount = 0;
                         return findPosition;
                     }
+
+                    Long timestamp = logPosition.getPostion().getTimestamp();
+                    if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
+                        // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
+                        return null;
+                    }
                 }
                 // 其余情况
                 logger.warn("prepare to find start position just last position\n {}",
@@ -745,14 +752,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                             logPosition.setPostion(entryPosition);
                         }
 
-                        if (entry == null) {
+                        // 直接用event的位点来处理,解决一个binlog文件里没有任何事件导致死循环无法退出的问题
-                            return true;
+                        String logfilename = event.getHeader().getLogFileName();
-                        }
+                        // 记录的是binlog end offest,
-
+                        // 因为与其对比的offest是show master status里的end offest
-                        String logfilename = entry.getHeader().getLogfileName();
+                        Long logfileoffset = event.getHeader().getLogPos();
-                        Long logfileoffset = entry.getHeader().getLogfileOffset();
+                        Long logposTimestamp = event.getHeader().getWhen() * 1000;
-                        Long logposTimestamp = entry.getHeader().getExecuteTime();
+                        Long serverId = event.getHeader().getServerId();
-                        Long serverId = entry.getHeader().getServerId();
 
                         // 如果最小的一条记录都不满足条件，可直接退出
                         if (logposTimestamp >= startTimestamp) {
@@ -764,6 +770,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                             return false;
                         }
 
+                        if (entry == null) {
+                            return true;
+                        }
+
                         // 记录一下上一个事务结束的位置，即下一个事务的position
                         // position = current +
                         // data.length，代表该事务的下一条offest，避免多余的事务重复
@@ -905,4 +915,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         this.dumpErrorCountThreshold = dumpErrorCountThreshold;
     }
 
+    public boolean isRdsOssMode() {
+        return rdsOssMode;
+    }
+
+    public void setRdsOssMode(boolean rdsOssMode) {
+        this.rdsOssMode = rdsOssMode;
+    }
+
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java

@@ -33,6 +33,7 @@ import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
@@ -69,6 +70,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
     private WorkerPool<MessageEvent>          workerPool;
     private BatchEventProcessor<MessageEvent> simpleParserStage;
     private BatchEventProcessor<MessageEvent> sinkStoreStage;
+    private LogContext                        logContext;
 
     public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
                                       EventTransactionBuffer transactionBuffer, String destination){
@@ -95,9 +97,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         SequenceBarrier sequenceBarrier = disruptorMsgBuffer.newBarrier();
         ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
         // stage 2
+        this.logContext = new LogContext();
         simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
             sequenceBarrier,
-            new SimpleParserStage());
+            new SimpleParserStage(logContext));
         simpleParserStage.setExceptionHandler(exceptionHandler);
         disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence());
 
@@ -128,6 +131,12 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         workerPool.start(parserExecutor);
     }
 
+    public void setBinlogChecksum(int binlogChecksum) {
+        if (binlogChecksum != LogEvent.BINLOG_CHECKSUM_ALG_OFF) {
+            logContext.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
+        }
+    }
+
     @Override
     public void stop() {
         // fix bug #968，对于pool与
@@ -239,9 +248,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         private LogDecoder decoder;
         private LogContext context;
 
-        public SimpleParserStage(){
+        public SimpleParserStage(LogContext context){
             decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
-            context = new LogContext();
+            this.context = context;
             if (gtidSet != null) {
                 context.setGtidSet(gtidSet);
             }
@@ -266,6 +275,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
                         needDmlParse = true;
                         break;
                     case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                     case LogEvent.UPDATE_ROWS_EVENT:
                         tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
                         needDmlParse = true;
@@ -468,4 +478,5 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
     public void setGtidSet(GTIDSet gtidSet) {
         this.gtidSet = gtidSet;
     }
+
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java

@@ -120,6 +120,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
             case LogEvent.WRITE_ROWS_EVENT:
                 return parseRowsEvent((WriteRowsLogEvent) logEvent);
             case LogEvent.UPDATE_ROWS_EVENT_V1:
+            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
             case LogEvent.UPDATE_ROWS_EVENT:
                 return parseRowsEvent((UpdateRowsLogEvent) logEvent);
             case LogEvent.DELETE_ROWS_EVENT_V1:
@@ -260,7 +261,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 return null;
             }
 
-            if (!isSeek) {
+            boolean isDml = (type == EventType.INSERT || type == EventType.UPDATE || type == EventType.DELETE);
+
+            if (!isSeek && !isDml) {
                 // 使用新的表结构元数据管理方式
                 EntryPosition position = createPosition(event.getHeader());
                 tableMetaCache.apply(position, event.getDbName(), queryString, null);
@@ -268,8 +271,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
             Header header = createHeader(event.getHeader(), schemaName, tableName, type);
             RowChange.Builder rowChangeBuider = RowChange.newBuilder();
-            if (type != EventType.QUERY && type != EventType.INSERT && type != EventType.UPDATE
+            if (type != EventType.QUERY && !isDml) {
-                && type != EventType.DELETE) {
                 rowChangeBuider.setIsDdl(true);
             }
             rowChangeBuider.setSql(queryString);
@@ -503,7 +505,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
             int type = event.getHeader().getType();
             if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                 eventType = EventType.INSERT;
-            } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
+            } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type
+                       || LogEvent.PARTIAL_UPDATE_ROWS_EVENT == type) {
                 eventType = EventType.UPDATE;
             } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                 eventType = EventType.DELETE;
@@ -522,7 +525,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
             boolean tableError = false;
             int rowsCount = 0;
-            while (buffer.nextOneRow(columns)) {
+            while (buffer.nextOneRow(columns, false)) {
                 // 处理row记录
                 RowData.Builder rowDataBuilder = RowData.newBuilder();
                 if (EventType.INSERT == eventType) {
@@ -534,7 +537,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 } else {
                     // update需要处理before/after
                     tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
-                    if (!buffer.nextOneRow(changeColumns)) {
+                    if (!buffer.nextOneRow(changeColumns, true)) {
                         rowChangeBuider.addRowDatas(rowDataBuilder.build());
                         break;
                     }
@@ -567,8 +570,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     }
 
     private EntryPosition createPosition(LogHeader logHeader) {
-        return new EntryPosition(logHeader.getLogFileName(),
+        return new EntryPosition(logHeader.getLogFileName(), logHeader.getLogPos() - logHeader.getEventLen(), // startPos
-            logHeader.getLogPos(),
             logHeader.getWhen() * 1000L,
             logHeader.getServerId()); // 记录到秒
     }
@@ -577,7 +579,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                                 boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
         int columnCnt = event.getTable().getColumnCnt();
         ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
-
+        // mysql8.0针对set @@global.binlog_row_metadata='FULL' 可以记录部分的metadata信息
+        boolean existOptionalMetaData = event.getTable().isExistOptionalMetaData();
         boolean tableError = false;
         // check table fileds count，只能处理加字段
         boolean existRDSNoPrimaryKey = false;
@@ -631,22 +634,43 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 continue;
             }
 
-            if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
-                // 不解析最后一列
-                buffer.nextValue(info.type, info.meta, false);
-                continue;
-            }
-
-            Column.Builder columnBuilder = Column.newBuilder();
-
             FieldMeta fieldMeta = null;
             if (tableMeta != null && !tableError) {
                 // 处理file meta
                 fieldMeta = tableMeta.getFields().get(i);
+            }
+
+            if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
+                // 不解析最后一列
+                buffer.nextValue(fieldMeta.getColumnName(), i, info.type, info.meta, false);
+                continue;
+            }
+
+            if (fieldMeta != null && existOptionalMetaData) {
+                // check column info
+                boolean check = StringUtils.equalsIgnoreCase(fieldMeta.getColumnName(), info.name);
+                check &= (fieldMeta.isUnsigned() == info.unsigned);
+                check &= (fieldMeta.isNullable() == info.nullable);
+
+                if (!check) {
+                    throw new CanalParseException("MySQL8.0 unmatch column metadata & pls submit issue , db : "
+                                                  + fieldMeta.toString() + " , binlog : " + info.toString()
+                                                  + " , on : " + event.getHeader().getLogFileName() + ":"
+                                                  + event.getHeader().getLogPos());
+                }
+            }
+
+            Column.Builder columnBuilder = Column.newBuilder();
+            if (fieldMeta != null) {
                 columnBuilder.setName(fieldMeta.getColumnName());
                 columnBuilder.setIsKey(fieldMeta.isKey());
                 // 增加mysql type类型,issue 73
                 columnBuilder.setMysqlType(fieldMeta.getColumnType());
+            } else if (existOptionalMetaData) {
+                columnBuilder.setName(info.name);
+                columnBuilder.setIsKey(info.pk);
+                // mysql8.0里没有mysql type类型
+                // columnBuilder.setMysqlType(fieldMeta.getColumnType());
             }
             columnBuilder.setIndex(i);
             columnBuilder.setIsNull(false);
@@ -664,12 +688,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                     isSingleBit = true;
                 }
             }
-            buffer.nextValue(info.type, info.meta, isBinary);
-            if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
-                // 不解析最后一列
-                continue;
-            }
 
+            buffer.nextValue(fieldMeta.getColumnName(), i, info.type, info.meta, isBinary);
             int javaType = buffer.getJavaType();
             if (buffer.isNull()) {
                 columnBuilder.setIsNull(true);
@@ -683,7 +703,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                     case Types.BIGINT:
                         // 处理unsigned类型
                         Number number = (Number) value;
-                        if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
+                        boolean isUnsigned = (fieldMeta != null ? fieldMeta.isUnsigned() : (existOptionalMetaData ? info.unsigned : false));
+                        if (isUnsigned && number.longValue() < 0) {
                             switch (buffer.getLength()) {
                                 case 1: /* MYSQL_TYPE_TINY */
                                     columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE
@@ -772,7 +793,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                     default:
                         columnBuilder.setValue(value.toString());
                 }
-
             }
 
             columnBuilder.setSqlType(javaType);
@@ -818,6 +838,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
         Header.Builder headerBuilder = Header.newBuilder();
         headerBuilder.setVersion(version);
         headerBuilder.setLogfileName(logHeader.getLogFileName());
+        // 记录的是该binlog的start offest
         headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
         headerBuilder.setServerId(logHeader.getServerId());
         headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
@@ -981,4 +1002,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     public void setFilterRows(boolean filterRows) {
         this.filterRows = filterRows;
     }
+
+    public void setUseDruidDdlFilter(boolean useDruidDdlFilter) {
+        this.useDruidDdlFilter = useDruidDdlFilter;
+    }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java

@@ -41,6 +41,7 @@ import org.apache.http.ssl.TrustStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
 
 /**
@@ -103,6 +104,9 @@ public class BinlogDownloadQueue {
 
     public BinlogFile tryOne() throws Throwable {
         BinlogFile binlogFile = binlogList.poll();
+        if (binlogFile == null) {
+            throw new CanalParseException("download binlog is null");
+        }
         download(binlogFile);
         hostId = binlogFile.getHostInstanceID();
         this.currentSize++;
@@ -131,7 +135,7 @@ public class BinlogDownloadQueue {
         if (StringUtils.isNotEmpty(needCompareName) && StringUtils.endsWith(needCompareName, "tar")) {
             needCompareName = needCompareName.substring(0, needCompareName.indexOf("."));
         }
-        return fileName.equalsIgnoreCase(needCompareName) && binlogList.isEmpty();
+        return (needCompareName == null || fileName.equalsIgnoreCase(needCompareName)) && binlogList.isEmpty();
     }
 
     public void prepare() throws InterruptedException {
@@ -162,6 +166,14 @@ public class BinlogDownloadQueue {
         this.currentSize = 0;
         binlogList.clear();
         downloadQueue.clear();
+        try {
+            downloadThread.interrupt();
+            downloadThread.join();// 等待其结束
+        } catch (InterruptedException e) {
+            // ignore
+        } finally {
+            downloadThread = null;
+        }
     }
 
     private void download(BinlogFile binlogFile) throws Throwable {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java

@@ -22,14 +22,14 @@ import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
  */
 public class RdsBinlogEventParserProxy extends MysqlEventParser {
 
-    private String                    rdsOpenApiUrl             = "https://rds.aliyuncs.com/";    // openapi地址
+    private String                    rdsOpenApiUrl             = "https://rds.aliyuncs.com/"; // openapi地址
-    private String                    accesskey;                                                  // 云账号的ak
+    private String                    accesskey;                                              // 云账号的ak
-    private String                    secretkey;                                                  // 云账号sk
+    private String                    secretkey;                                              // 云账号sk
-    private String                    instanceId;                                                 // rds实例id
+    private String                    instanceId;                                             // rds实例id
-    private String                    directory;                                                  // binlog目录
+    private String                    directory;                                              // binlog目录
-    private int                       batchFileSize             = 4;                              // 最多下载的binlog文件数量
+    private int                       batchFileSize             = 4;                          // 最多下载的binlog文件数量
 
-    private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser();
+    private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = null;
     private ExecutorService           executorService           = Executors.newSingleThreadExecutor(new ThreadFactory() {
 
                                                                     @Override
@@ -43,8 +43,11 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
 
     @Override
     public void start() {
-        if (StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey)
+        if (rdsLocalBinlogEventParser == null && StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey)
             && StringUtils.isNotEmpty(instanceId)) {
+            rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser();
+            // rds oss mode
+            setRdsOssMode(true);
             final ParserExceptionHandler targetHandler = this.getParserExceptionHandler();
             if (directory == null) {
                 directory = System.getProperty("java.io.tmpdir", "/tmp") + "/" + destination;
@@ -119,10 +122,18 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
                         long serverId = rdsBinlogEventParserProxy.getServerId();
                         rdsLocalBinlogEventParser.setServerId(serverId);
                         rdsBinlogEventParserProxy.stop();
+                    } catch (Throwable e) {
+                        logger.info("handle exception failed", e);
+                    }
+
+                    try {
                         logger.info("start rds mysql binlog parser!");
                         rdsLocalBinlogEventParser.start();
                     } catch (Throwable e) {
                         logger.info("handle exception failed", e);
+                        rdsLocalBinlogEventParser.stop();
+                        RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
+                        rdsBinlogEventParserProxy.start();// 继续重试
                     }
                 }
             });

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java

@@ -57,7 +57,11 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
             if (entryPosition == null) {
                 throw new PositionNotFoundException("position not found!");
             }
-            long startTimeInMill = entryPosition.getTimestamp();
+            Long startTimeInMill = entryPosition.getTimestamp();
+            if (startTimeInMill == null || startTimeInMill <= 0) {
+                throw new PositionNotFoundException("position timestamp is empty!");
+            }
+
             startTime = startTimeInMill;
             List<BinlogFile> binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url,
                 accesskey,
@@ -65,6 +69,10 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
                 instanceId,
                 new Date(startTime),
                 new Date(endTime));
+            if (binlogFiles.isEmpty()) {
+                throw new CanalParseException("start timestamp : " + startTimeInMill + " binlog files is empty");
+            }
+
             binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchFileSize, directory);
             binlogDownloadQueue.silenceDownload();
             needWait = true;

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java

@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
@@ -48,17 +49,25 @@ public class DatabaseTableMeta implements TableMetaTSDB {
     private static Logger             logger           = LoggerFactory.getLogger(DatabaseTableMeta.class);
     private static Pattern            pattern          = Pattern.compile("Duplicate entry '.*' for key '*'");
     private static Pattern            h2Pattern        = Pattern.compile("Unique index or primary key violation");
+    private static ScheduledExecutorService  scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+        @Override
+        public Thread newThread(Runnable r) {
+            Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]");
+            thread.setDaemon(true);
+            return thread;
+        }
+    });
     private String                    destination;
     private MemoryTableMeta           memoryTableMeta;
     private MysqlConnection           connection;                                                                 // 查询meta信息的链接
     private CanalEventFilter          filter;
     private CanalEventFilter          blackFilter;
     private EntryPosition             lastPosition;
-    private ScheduledExecutorService  scheduler;
     private MetaHistoryDAO            metaHistoryDAO;
     private MetaSnapshotDAO           metaSnapshotDAO;
     private int                       snapshotInterval = 24;
     private int                       snapshotExpire   = 360;
+    private ScheduledFuture<?>        scheduleSnapshotFuture;
     
     public DatabaseTableMeta(){
 
@@ -68,19 +77,10 @@ public class DatabaseTableMeta implements TableMetaTSDB {
     public boolean init(final String destination) {
         this.destination = destination;
         this.memoryTableMeta = new MemoryTableMeta();
-        this.scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
-
-            @Override
-            public Thread newThread(Runnable r) {
-                Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]");
-                thread.setDaemon(true);
-                return thread;
-            }
-        });
 
         // 24小时生成一份snapshot
         if (snapshotInterval > 0) {
-            scheduler.scheduleWithFixedDelay(new Runnable() {
+            scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() {
 
                 @Override
                 public void run() {
@@ -106,6 +106,26 @@ public class DatabaseTableMeta implements TableMetaTSDB {
         return true;
     }
     
+    @Override
+    public void destory() {
+        if (memoryTableMeta != null) {
+            memoryTableMeta.destory();
+        }
+        
+        if (connection != null) {
+            try {
+                connection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect meta connection for address:{}", connection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+        
+        if (scheduleSnapshotFuture != null) {
+            scheduleSnapshotFuture.cancel(false);
+        }
+    }
+
     @Override
     public TableMeta find(String schema, String table) {
         synchronized (memoryTableMeta) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java

@@ -59,6 +59,11 @@ public class MemoryTableMeta implements TableMetaTSDB {
         return true;
     }
     
+    @Override
+    public void destory() {
+        tableMetas.clear();
+    }
+
     public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
         tableMetas.clear();
         synchronized (this) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java

@@ -18,6 +18,11 @@ public interface TableMetaTSDB {
      */
     public boolean init(String destination);
 
+    /**
+     * 销毁资源
+     */
+    public void destory();
+
     /**
      * 获取当前的表结构
      */

parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java

@@ -1,7 +1,12 @@
 package com.alibaba.otter.canal.parse;
 
 import java.io.IOException;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
 import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.BitSet;
+import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
 import org.junit.Assert;
@@ -10,29 +15,49 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
 import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 
 public class DirectLogFetcherTest {
 
     protected final Logger logger = LoggerFactory.getLogger(this.getClass());
+    protected String       binlogFileName = "mysql-bin.000001";
+    protected Charset      charset        = Charset.forName("utf-8");
+    private boolean        isMariaDB;
+    private int            binlogChecksum;
 
     @Test
     public void testSimple() {
         DirectLogFetcher fetcher = new DirectLogFetcher();
         try {
-            MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxx", "xxxx");
+            MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
             connector.connect();
             updateSettings(connector);
+            loadBinlogChecksum(connector);
             sendRegisterSlave(connector, 3);
             sendBinlogDump(connector, "mysql-bin.000001", 4L, 3);
 
@@ -40,6 +65,8 @@ public class DirectLogFetcherTest {
 
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
+            context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
+
             while (fetcher.fetch()) {
                 LogEvent event = null;
                 event = decoder.decode(fetcher, context);
@@ -57,21 +84,22 @@ public class DirectLogFetcherTest {
                         break;
                     case LogEvent.WRITE_ROWS_EVENT_V1:
                     case LogEvent.WRITE_ROWS_EVENT:
-                        // parseRowsEvent((WriteRowsLogEvent) event);
+                        parseRowsEvent((WriteRowsLogEvent) event);
                         break;
                     case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                     case LogEvent.UPDATE_ROWS_EVENT:
-                        // parseRowsEvent((UpdateRowsLogEvent) event);
+                        parseRowsEvent((UpdateRowsLogEvent) event);
                         break;
                     case LogEvent.DELETE_ROWS_EVENT_V1:
                     case LogEvent.DELETE_ROWS_EVENT:
-                        // parseRowsEvent((DeleteRowsLogEvent) event);
+                        parseRowsEvent((DeleteRowsLogEvent) event);
                         break;
                     case LogEvent.QUERY_EVENT:
-                        // parseQueryEvent((QueryLogEvent) event);
+                        parseQueryEvent((QueryLogEvent) event);
                         break;
                     case LogEvent.ROWS_QUERY_LOG_EVENT:
-                        // parseRowsQueryEvent((RowsQueryLogEvent) event);
+                        parseRowsQueryEvent((RowsQueryLogEvent) event);
                         break;
                     case LogEvent.ANNOTATE_ROWS_EVENT:
                         break;
@@ -188,9 +216,141 @@ public class DirectLogFetcherTest {
         }
     }
 
+    private void loadBinlogChecksum(MysqlConnector connector) {
+        checkMariaDB(connector);
+        if (isMariaDB) {
+            ResultSetPacket rs = null;
+            try {
+                rs = query("select @@global.binlog_checksum", connector);
+            } catch (IOException e) {
+                throw new CanalParseException(e);
+            }
+
+            List<String> columnValues = rs.getFieldValues();
+            if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
+                binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
+            } else {
+                binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
+            }
+        }
+    }
+
+    private void checkMariaDB(MysqlConnector connector) {
+        ResultSetPacket rs = null;
+        try {
+            rs = query("SELECT @@version", connector);
+        } catch (IOException e) {
+            throw new CanalParseException(e);
+        }
+
+        List<String> columnValues = rs.getFieldValues();
+        if (columnValues != null && columnValues.size() >= 1) {
+            isMariaDB = StringUtils.containsIgnoreCase(columnValues.get(0), "MariaDB");
+        }
+    }
+
+    public ResultSetPacket query(String cmd, MysqlConnector connector) throws IOException {
+        MysqlQueryExecutor exector = new MysqlQueryExecutor(connector);
+        return exector.query(cmd);
+    }
+
     public void update(String cmd, MysqlConnector connector) throws IOException {
         MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
         exector.update(cmd);
     }
 
+    protected void parseQueryEvent(QueryLogEvent event) {
+        System.out.println(String.format("================> binlog[%s:%s] , name[%s]",
+            binlogFileName,
+            event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+            event.getCatalog()));
+        System.out.println("sql : " + event.getQuery());
+    }
+
+    protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
+    }
+
+    protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
+    }
+
+    protected void parseXidEvent(XidLogEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
+            .getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("xid : " + event.getXid());
+    }
+
+    protected void parseRowsEvent(RowsLogEvent event) {
+        try {
+            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
+                binlogFileName,
+                event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+                event.getTable().getDbName(),
+                event.getTable().getTableName()));
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+            BitSet columns = event.getColumns();
+            BitSet changeColumns = event.getChangeColumns();
+            while (buffer.nextOneRow(columns)) {
+                // 处理row记录
+                int type = event.getHeader().getType();
+                if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
+                    // insert的记录放在before字段中
+                    parseOneRow(event, buffer, columns, true);
+                } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
+                    // delete的记录放在before字段中
+                    parseOneRow(event, buffer, columns, false);
+                } else {
+                    // update需要处理before/after
+                    System.out.println("-------> before");
+                    parseOneRow(event, buffer, columns, false);
+                    if (!buffer.nextOneRow(changeColumns, true)) {
+                        break;
+                    }
+                    System.out.println("-------> after");
+                    parseOneRow(event, buffer, changeColumns, true);
+                }
+
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("parse row data failed.", e);
+        }
+    }
+
+    protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
+                                                                                                      throws UnsupportedEncodingException {
+        TableMapLogEvent map = event.getTable();
+        if (map == null) {
+            throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
+        }
+
+        final int columnCnt = map.getColumnCnt();
+        final ColumnInfo[] columnInfo = map.getColumnInfo();
+
+        for (int i = 0; i < columnCnt; i++) {
+            if (!cols.get(i)) {
+                continue;
+            }
+
+            ColumnInfo info = columnInfo[i];
+            buffer.nextValue(null, i, info.type, info.meta);
+
+            if (buffer.isNull()) {
+                //
+            } else {
+                final Serializable value = buffer.getValue();
+                if (value instanceof byte[]) {
+                    System.out.println(new String((byte[]) value));
+                } else {
+                    System.out.println(value);
+                }
+            }
+        }
+
+    }
+
 }

parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java

@@ -89,6 +89,7 @@ public class MysqlBinlogParsePerformanceTest {
                     parseRowsEvent((WriteRowsLogEvent) event, sum);
                     break;
                 case LogEvent.UPDATE_ROWS_EVENT_V1:
+                case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                 case LogEvent.UPDATE_ROWS_EVENT:
                     parseRowsEvent((UpdateRowsLogEvent) event, sum);
                     break;
@@ -154,7 +155,7 @@ public class MysqlBinlogParsePerformanceTest {
                     parseOneRow(event, buffer, columns, false);
                 } else {
                     parseOneRow(event, buffer, columns, false);
-                    if (!buffer.nextOneRow(changeColumns)) {
+                    if (!buffer.nextOneRow(changeColumns, true)) {
                         break;
                     }
                     parseOneRow(event, buffer, changeColumns, true);
@@ -182,7 +183,7 @@ public class MysqlBinlogParsePerformanceTest {
             }
 
             ColumnInfo info = columnInfo[i];
-            buffer.nextValue(info.type, info.meta);
+            buffer.nextValue(null, i, info.type, info.meta);
             if (buffer.isNull()) {
             } else {
                 buffer.getValue();

parse/src/test/resources/tsdb/h2-tsdb.xml

@@ -7,7 +7,7 @@
        default-autowire="byName">
 	
 	<!-- 基于db的实现 -->
-	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
 		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
 		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
 	</bean>

parse/src/test/resources/tsdb/mysql-tsdb.xml

@@ -7,7 +7,7 @@
        default-autowire="byName">
 	
 	<!-- 基于db的实现 -->
-	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
+	<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta" destroy-method="destory">
 		<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
 		<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
 	</bean>

pom.xml

@@ -371,6 +371,7 @@
                 </configuration>
             </plugin>
             <!-- javadoc -->
+            <!--
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
@@ -389,61 +390,7 @@
                   <additionalparam>-Xdoclint:none</additionalparam>
                 </configuration>
             </plugin>
-            <!--
+            -->
-            <plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-javadoc-plugin</artifactId>
-				<version>2.7</version>
-				<executions>
-					<execution>
-						<id>attach-javadocs</id>
-							<goals>
-								<goal>jar</goal>
-							</goals>
-						</execution>
-					</executions>
-				<configuration>
-				  <encoding>${file_encoding}</encoding>
-				  <charset>${file_encoding}</charset>
-				  <doclet>org.jboss.apiviz.APIviz</doclet>
-				  <docletArtifact>
-					<groupId>org.jboss.apiviz</groupId>
-					<artifactId>apiviz</artifactId>
-					<version>1.3.0.GA</version>
-				  </docletArtifact>
-				  <useStandardDocletOptions>true</useStandardDocletOptions>
-				  <breakiterator>true</breakiterator>
-				  <version>true</version>
-				  <author>true</author>
-				  <keywords>true</keywords>
-				</configuration>
-			</plugin>
-
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-jxr-plugin</artifactId>
-				<version>2.2</version>
-				<configuration>
-					<aggregate>true</aggregate>
-					<destDir>${project.basedir}/docs/sources</destDir>
-					<linkJavadoc>true</linkJavadoc>
-					<javadocDir>${project.basedir}/docs/javadoc</javadocDir>
-				</configuration>
-			</plugin>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-source-plugin</artifactId>
-				<version>2.1.2</version>
-				<executions>
-					<execution>
-					<id>attach-sources</id>
-						<goals>
-							<goal>jar</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
-			-->
         </plugins>
         <sourceDirectory>src/main/java</sourceDirectory>
         <testSourceDirectory>src/test/java</testSourceDirectory>

server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java

@@ -16,7 +16,7 @@ import com.google.protobuf.WireFormat;
 public class CanalMessageSerializer {
 
     @SuppressWarnings("deprecation")
-    public static byte[] serializer(Message data) {
+    public static byte[] serializer(Message data, boolean filterTransactionEntry) {
         try {
             if (data != null) {
                 if (data.getId() != -1) {
@@ -53,8 +53,14 @@ public class CanalMessageSerializer {
                         output.checkNoSpaceLeft();
                         return body;
                     } else if (!CollectionUtils.isEmpty(data.getEntries())) {
+                        // mq模式只会走到非rowEntry模式
                         CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                         for (CanalEntry.Entry entry : data.getEntries()) {
+                            if (filterTransactionEntry
+                                && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
+                                continue;
+                            }
+
                             messageBuilder.addMessages(entry.toByteString());
                         }
 

server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.kafka;
 
 import java.util.Map;
 
+import org.apache.commons.lang.BooleanUtils;
 import org.apache.kafka.common.serialization.Serializer;
 
 import com.alibaba.otter.canal.common.CanalMessageSerializer;
@@ -15,13 +16,20 @@ import com.alibaba.otter.canal.protocol.Message;
  */
 public class MessageSerializer implements Serializer<Message> {
 
+    private boolean filterTransactionEntry = false;
+
+    public MessageSerializer(){
+        this.filterTransactionEntry = BooleanUtils.toBoolean(System.getProperty("canal.instance.filter.transaction.entry",
+            "false"));
+    }
+
     @Override
     public void configure(Map<String, ?> configs, boolean isKey) {
     }
 
     @Override
     public byte[] serialize(String topic, Message data) {
-        return CanalMessageSerializer.serializer(data);
+        return CanalMessageSerializer.serializer(data, filterTransactionEntry);
     }
 
     @Override

server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java

@@ -47,7 +47,8 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                      Callback callback) {
         if (!mqProperties.getFlatMessage()) {
             try {
-                Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data));
+                Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data,
+                    mqProperties.isFilterTransactionEntry()));
                 logger.debug("send message:{} to destination:{}, partition: {}",
                     message,
                     destination.getCanalDestination(),
@@ -77,8 +78,8 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                                 JSON.toJSONString(flatMessage),
                                 destination.getTopic(),
                                 destination.getPartition());
-                            Message message = new Message(destination.getTopic(), JSON.toJSONString(flatMessage)
+                            Message message = new Message(destination.getTopic(),
-                                .getBytes());
+                                JSON.toJSONString(flatMessage).getBytes());
                             this.defaultMQProducer.send(message, new MessageQueueSelector() {
 
                                 @Override
@@ -98,28 +99,32 @@ public class CanalRocketMQProducer implements CanalMQProducer {
                             int length = partitionFlatMessage.length;
                             for (int i = 0; i < length; i++) {
                                 FlatMessage flatMessagePart = partitionFlatMessage[i];
-                                logger.debug("flatMessagePart: {}, partition: {}",
+                                if (flatMessagePart != null) {
-                                    JSON.toJSONString(flatMessagePart),
+                                    logger.debug("flatMessagePart: {}, partition: {}",
-                                    i);
+                                        JSON.toJSONString(flatMessagePart),
-                                final int index = i;
+                                        i);
-                                try {
+                                    final int index = i;
-                                    Message message = new Message(destination.getTopic(),
+                                    try {
-                                        JSON.toJSONString(flatMessagePart).getBytes());
+                                        Message message = new Message(destination.getTopic(),
-                                    this.defaultMQProducer.send(message, new MessageQueueSelector() {
+                                            JSON.toJSONString(flatMessagePart).getBytes());
+                                        this.defaultMQProducer.send(message, new MessageQueueSelector() {
 
-                                        @Override
+                                            @Override
-                                        public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
+                                            public MessageQueue select(List<MessageQueue> mqs, Message msg,
-                                            if (index > mqs.size()) {
+                                                                       Object arg) {
-                                                throw new CanalServerException("partition number is error,config num:"
+                                                if (index > mqs.size()) {
-                                                                               + destination.getPartitionsNum()
+                                                    throw new CanalServerException(
-                                                                               + ", mq num: " + mqs.size());
+                                                        "partition number is error,config num:"
+                                                                                   + destination.getPartitionsNum()
+                                                                                   + ", mq num: " + mqs.size());
+                                                }
+                                                return mqs.get(index);
                                             }
-                                            return mqs.get(index);
+                                        }, null);
-                                        }
+                                    } catch (Exception e) {
-                                    }, null);
+                                        logger.error("send flat message to hashed partition error", e);
-                                } catch (Exception e) {
+                                        callback.rollback();
-                                    logger.error("send flat message to hashed partition error", e);
+                                    }
-                                    callback.rollback();
                                 }
                             }
                         }