模块化mq connector (#2562)

* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module

connector/core/pom.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.core</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector core module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.googlecode.aviator</groupId>
+            <artifactId>aviator</artifactId>
+            <version>2.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>oro</groupId>
+            <artifactId>oro</artifactId>
+            <version>2.0.8</version>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+    </dependencies>
+
+</project>

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java

@@ -0,0 +1,26 @@
+package com.alibaba.otter.canal.connector.core.config;
+
+/**
+ * MQ配置常量
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class CanalConstants {
+
+    public static final String ROOT                           = "canal";
+
+    public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
+
+    public static final String CANAL_MQ_FLAT_MESSAGE          = ROOT + "." + "mq.flat.message";
+    public static final String CANAL_MQ_DATABASE_HASH         = ROOT + "." + "mq.database.hash";
+    public static final String CANAL_MQ_PARALLEL_THREAD_SIZE  = ROOT + "." + "mq.parallel.thread.size";
+    public static final String CANAL_MQ_CANAL_BATCH_SIZE      = ROOT + "." + "mq.canal.batch.size";
+    public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT   = ROOT + "." + "mq.canal.fetch.timeout";
+    public static final String CANAL_MQ_ACCESS_CHANNEL        = ROOT + "." + "mq.access.channel";
+
+    public static final String CANAL_ALIYUN_ACCESS_KEY        = ROOT + "." + "aliyun.accessKey";
+    public static final String CANAL_ALIYUN_SECRET_KEY        = ROOT + "." + "aliyun.secretKey";
+    public static final String CANAL_ALIYUN_UID               = ROOT + "." + "aliyun.uid";
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java

@@ -0,0 +1,102 @@
+package com.alibaba.otter.canal.connector.core.config;
+
+/**
+ * MQ配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class MQProperties {
+
+    private boolean flatMessage            = true;
+    private boolean databaseHash           = true;
+    private boolean filterTransactionEntry = true;
+    private Integer parallelThreadSize     = 8;
+    private Integer fetchTimeout           = 100;
+    private Integer batchSize              = 50;
+    private String  accessChannel          = "local";
+
+    private String  aliyunAccessKey        = "";
+    private String  aliyunSecretKey        = "";
+    private int     aliyunUid              = 0;
+
+    public boolean isFlatMessage() {
+        return flatMessage;
+    }
+
+    public void setFlatMessage(boolean flatMessage) {
+        this.flatMessage = flatMessage;
+    }
+
+    public boolean isDatabaseHash() {
+        return databaseHash;
+    }
+
+    public void setDatabaseHash(boolean databaseHash) {
+        this.databaseHash = databaseHash;
+    }
+
+    public boolean isFilterTransactionEntry() {
+        return filterTransactionEntry;
+    }
+
+    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
+        this.filterTransactionEntry = filterTransactionEntry;
+    }
+
+    public Integer getParallelThreadSize() {
+        return parallelThreadSize;
+    }
+
+    public void setParallelThreadSize(Integer parallelThreadSize) {
+        this.parallelThreadSize = parallelThreadSize;
+    }
+
+    public Integer getFetchTimeout() {
+        return fetchTimeout;
+    }
+
+    public void setFetchTimeout(Integer fetchTimeout) {
+        this.fetchTimeout = fetchTimeout;
+    }
+
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(Integer batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public String getAccessChannel() {
+        return accessChannel;
+    }
+
+    public void setAccessChannel(String accessChannel) {
+        this.accessChannel = accessChannel;
+    }
+
+    public String getAliyunAccessKey() {
+        return aliyunAccessKey;
+    }
+
+    public void setAliyunAccessKey(String aliyunAccessKey) {
+        this.aliyunAccessKey = aliyunAccessKey;
+    }
+
+    public String getAliyunSecretKey() {
+        return aliyunSecretKey;
+    }
+
+    public void setAliyunSecretKey(String aliyunSecretKey) {
+        this.aliyunSecretKey = aliyunSecretKey;
+    }
+
+    public int getAliyunUid() {
+        return aliyunUid;
+    }
+
+    public void setAliyunUid(int aliyunUid) {
+        this.aliyunUid = aliyunUid;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java

@@ -0,0 +1,131 @@
+package com.alibaba.otter.canal.connector.core.consumer;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public class CommonMessage implements Serializable {
+
+    private static final long serialVersionUID = 2611556444074013268L;
+
+    private String database;                               // 数据库或schema
+    private String table;                                  // 表名
+    private List<String> pkNames;
+    private Boolean isDdl;
+    private String type;                                   // 类型: INSERT UPDATE DELETE
+    // binlog executeTime
+    private Long es;                                       // 执行耗时
+    // dml build timeStamp
+    private Long ts;                                       // 同步时间
+    private String sql;                                    // 执行的sql, dml sql为空
+    private List<Map<String, Object>> data;                // 数据列表
+    private List<Map<String, Object>> old;                 // 旧数据列表, 用于update, size和data的size一一对应
+
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public List<String> getPkNames() {
+        return pkNames;
+    }
+
+    public void setPkNames(List<String> pkNames) {
+        this.pkNames = pkNames;
+    }
+
+    public Boolean getIsDdl() {
+        return isDdl;
+    }
+
+    public void setIsDdl(Boolean isDdl) {
+        this.isDdl = isDdl;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Long getTs() {
+        return ts;
+    }
+
+    public void setTs(Long ts) {
+        this.ts = ts;
+    }
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    public List<Map<String, Object>> getData() {
+        return data;
+    }
+
+    public void setData(List<Map<String, Object>> data) {
+        this.data = data;
+    }
+
+    public List<Map<String, Object>> getOld() {
+        return old;
+    }
+
+    public void setOld(List<Map<String, Object>> old) {
+        this.old = old;
+    }
+
+    public Long getEs() {
+        return es;
+    }
+
+    public void setEs(Long es) {
+        this.es = es;
+    }
+
+    public void clear() {
+        database = null;
+        table = null;
+        type = null;
+        ts = null;
+        es = null;
+        data = null;
+        old = null;
+        sql = null;
+    }
+
+    @Override
+    public String toString() {
+        return "CommonMessage{" +
+                "database='" + database + '\'' +
+                ", table='" + table + '\'' +
+                ", pkNames=" + pkNames +
+                ", isDdl=" + isDdl +
+                ", type='" + type + '\'' +
+                ", es=" + es +
+                ", ts=" + ts +
+                ", sql='" + sql + '\'' +
+                ", data=" + data +
+                ", old=" + old +
+                '}';
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java

@@ -0,0 +1,125 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.googlecode.aviator.AviatorEvaluator;
+import com.googlecode.aviator.Expression;
+
+/**
+ * 基于aviater进行tableName正则匹配的过滤算法
+ *
+ * @author jianghang 2012-7-20 下午06:01:34
+ */
+public class AviaterRegexFilter {
+
+    private static final String        SPLIT             = ",";
+    private static final String        PATTERN_SPLIT     = "|";
+    private static final String        FILTER_EXPRESSION = "regex(pattern,target)";
+    private static final RegexFunction regexFunction     = new RegexFunction();
+    private final Expression           exp               = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
+    static {
+        AviatorEvaluator.addFunction(regexFunction);
+    }
+
+    private static final Comparator<String> COMPARATOR = new StringComparator();
+
+    final private String                    pattern;
+    final private boolean                   defaultEmptyValue;
+
+    public AviaterRegexFilter(String pattern){
+        this(pattern, true);
+    }
+
+    public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
+        this.defaultEmptyValue = defaultEmptyValue;
+        List<String> list = null;
+        if (StringUtils.isEmpty(pattern)) {
+            list = new ArrayList<String>();
+        } else {
+            String[] ss = StringUtils.split(pattern, SPLIT);
+            list = Arrays.asList(ss);
+        }
+
+        // 对pattern按照从长到短的排序
+        // 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
+        // 的长度不一样
+        Collections.sort(list, COMPARATOR);
+        // 对pattern进行头尾完全匹配
+        list = completionPattern(list);
+        this.pattern = StringUtils.join(list, PATTERN_SPLIT);
+    }
+
+    public boolean filter(String filtered) {
+        if (StringUtils.isEmpty(pattern)) {
+            return defaultEmptyValue;
+        }
+
+        if (StringUtils.isEmpty(filtered)) {
+            return defaultEmptyValue;
+        }
+
+        Map<String, Object> env = new HashMap<String, Object>();
+        env.put("pattern", pattern);
+        env.put("target", filtered.toLowerCase());
+        return (Boolean) exp.execute(env);
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
+     *
+     * <pre>
+     * foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
+     * </pre>
+     *
+     * 因此此类对正则表达式进行了从长到短的排序
+     *
+     * @author zebin.xuzb 2012-10-22 下午2:02:26
+     * @version 1.0.0
+     */
+    private static class StringComparator implements Comparator<String> {
+
+        @Override
+        public int compare(String str1, String str2) {
+            if (str1.length() > str2.length()) {
+                return -1;
+            } else if (str1.length() < str2.length()) {
+                return 1;
+            } else {
+                return 0;
+            }
+        }
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
+     *
+     * <pre>
+     * foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
+     * </pre>
+     *
+     * 因此此类对正则表达式进行头尾完全匹配
+     *
+     * @author simon
+     * @version 1.0.0
+     */
+
+    private List<String> completionPattern(List<String> patterns) {
+        List<String> result = new ArrayList<String>();
+        for (String pattern : patterns) {
+            StringBuffer stringBuffer = new StringBuffer();
+            stringBuffer.append("^");
+            stringBuffer.append(pattern);
+            stringBuffer.append("$");
+            result.add(stringBuffer.toString());
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return pattern;
+    }
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java

@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.MalformedPatternException;
+import org.apache.oro.text.regex.Pattern;
+import org.apache.oro.text.regex.PatternCompiler;
+import org.apache.oro.text.regex.Perl5Compiler;
+
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+
+public class PatternUtils {
+
+    @SuppressWarnings("deprecation")
+    private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> {
+        try {
+            PatternCompiler pc = new Perl5Compiler();
+            return pc.compile(pattern,
+                Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK);
+        } catch (MalformedPatternException e) {
+            throw new RuntimeException(e);
+        }
+    });
+
+    public static Pattern getPattern(String pattern) {
+        return patterns.get(pattern);
+    }
+
+    public static void clear() {
+        patterns.clear();
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.Perl5Matcher;
+
+import com.googlecode.aviator.runtime.function.AbstractFunction;
+import com.googlecode.aviator.runtime.function.FunctionUtils;
+import com.googlecode.aviator.runtime.type.AviatorBoolean;
+import com.googlecode.aviator.runtime.type.AviatorObject;
+
+/**
+ * 提供aviator regex的代码扩展
+ *
+ * @author jianghang 2012-7-23 上午10:29:23
+ */
+public class RegexFunction extends AbstractFunction {
+
+    public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
+        String pattern = FunctionUtils.getStringValue(arg1, env);
+        String text = FunctionUtils.getStringValue(arg2, env);
+        Perl5Matcher matcher = new Perl5Matcher();
+        boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
+        return AviatorBoolean.valueOf(isMatch);
+    }
+
+    public String getName() {
+        return "regex";
+    }
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java

@@ -0,0 +1,110 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+import java.util.Properties;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+
+/**
+ * MQ producer 抽象类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public abstract class AbstractMQProducer implements CanalMQProducer {
+
+    protected MQProperties       mqProperties;
+
+    protected ThreadPoolExecutor executor;
+
+    @Override
+    public void init(Properties properties) {
+        // parse canal mq properties
+        loadCanalMqProperties(properties);
+
+        int parallelThreadSize = mqProperties.getParallelThreadSize();
+        executor = new ThreadPoolExecutor(parallelThreadSize,
+            parallelThreadSize,
+            0,
+            TimeUnit.SECONDS,
+            new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
+            new NamedThreadFactory("MQParallel"),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+    }
+
+    @Override
+    public MQProperties getMqProperties() {
+        return this.mqProperties;
+    }
+
+    @Override
+    public void stop() {
+        if (executor != null) {
+            executor.shutdownNow();
+        }
+    }
+
+    /**
+     * 初始化配置
+     * <p>
+     * canal.mq.flat.message = true <br/>
+     * canal.mq.database.hash = true <br/>
+     * canal.mq.filter.transaction.entry = true <br/>
+     * canal.mq.parallel.thread.size = 8 <br/>
+     * canal.mq.batch.size = 50 <br/>
+     * canal.mq.timeout = 100 <br/>
+     * canal.mq.access.channel = local <br/>
+     * </p>
+     * 
+     * @param properties 总配置对象
+     */
+    private void loadCanalMqProperties(Properties properties) {
+        String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        if (!StringUtils.isEmpty(flatMessage)) {
+            mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
+        }
+        String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
+        if (!StringUtils.isEmpty(databaseHash)) {
+            mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
+        }
+        String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
+        if (!StringUtils.isEmpty(filterTranEntry)) {
+            mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
+        }
+        String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE);
+        if (!StringUtils.isEmpty(parallelThreadSize)) {
+            mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize));
+        }
+        String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
+        if (!StringUtils.isEmpty(batchSize)) {
+            mqProperties.setBatchSize(Integer.parseInt(batchSize));
+        }
+        String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT);
+        if (!StringUtils.isEmpty(timeOut)) {
+            mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
+        }
+        String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
+        if (!StringUtils.isEmpty(accessChannel)) {
+            mqProperties.setAccessChannel(accessChannel);
+        }
+        String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        if (!StringUtils.isEmpty(aliyunAccessKey)) {
+            mqProperties.setAliyunAccessKey(aliyunAccessKey);
+        }
+        String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        if (!StringUtils.isEmpty(aliyunSecretKey)) {
+            mqProperties.setAliyunAccessKey(aliyunSecretKey);
+        }
+        String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
+        if (!StringUtils.isEmpty(aliyunUid)) {
+            mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java

@@ -0,0 +1,64 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+/**
+ * MQ producer destination
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class MQDestination {
+    private String  canalDestination;
+    private String  topic;
+    private Integer partition;
+    private Integer partitionsNum;
+    private String  partitionHash;
+    private String  dynamicTopic;
+
+    public String getCanalDestination() {
+        return canalDestination;
+    }
+
+    public void setCanalDestination(String canalDestination) {
+        this.canalDestination = canalDestination;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public Integer getPartition() {
+        return partition;
+    }
+
+    public void setPartition(Integer partition) {
+        this.partition = partition;
+    }
+
+    public Integer getPartitionsNum() {
+        return partitionsNum;
+    }
+
+    public void setPartitionsNum(Integer partitionsNum) {
+        this.partitionsNum = partitionsNum;
+    }
+
+    public String getPartitionHash() {
+        return partitionHash;
+    }
+
+    public void setPartitionHash(String partitionHash) {
+        this.partitionHash = partitionHash;
+    }
+
+    public String getDynamicTopic() {
+        return dynamicTopic;
+    }
+
+    public void setDynamicTopic(String dynamicTopic) {
+        this.dynamicTopic = dynamicTopic;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java

@@ -0,0 +1,674 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+import java.util.*;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import com.google.common.base.Function;
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * process MQ Message utils
+ *
+ * @author agapple 2019年9月29日 下午12:36:26
+ * @since 5.0.0
+ */
+public class MQMessageUtils {
+
+    @SuppressWarnings("deprecation")
+    private static Map<String, List<PartitionData>>    partitionDatas    = MigrateMap
+        .makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> {
+            List<PartitionData> datas = Lists
+                .newArrayList();
+
+            String[] pkHashConfigArray = StringUtils
+                .split(StringUtils.replace(
+                    pkHashConfigs,
+                    ",",
+                    ";"), ";");
+            // schema.table:id^name
+            for (String pkHashConfig : pkHashConfigArray) {
+                PartitionData data = new PartitionData();
+                int i = pkHashConfig
+                    .lastIndexOf(":");
+                if (i > 0) {
+                    String pkStr = pkHashConfig
+                        .substring(i + 1);
+                    if (pkStr.equalsIgnoreCase(
+                        "$pk$")) {
+                        data.hashMode.autoPkHash = true;
+                    } else {
+                        data.hashMode.pkNames = Lists
+                            .newArrayList(
+                                StringUtils
+                                    .split(
+                                        pkStr,
+                                        '^'));
+                    }
+
+                    pkHashConfig = pkHashConfig
+                        .substring(0, i);
+                } else {
+                    data.hashMode.tableHash = true;
+                }
+
+                if (!isWildCard(
+                    pkHashConfig)) {
+                    data.simpleName = pkHashConfig;
+                } else {
+                    data.regexFilter = new AviaterRegexFilter(
+                        pkHashConfig);
+                }
+                datas.add(data);
+            }
+
+            return datas;
+        });
+
+    @SuppressWarnings("deprecation")
+    private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap
+        .makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() {
+
+                                                                                 public List<DynamicTopicData> apply(String pkHashConfigs) {
+                                                                                     List<DynamicTopicData> datas = Lists
+                                                                                         .newArrayList();
+                                                                                     String[] dynamicTopicArray = StringUtils
+                                                                                         .split(StringUtils.replace(
+                                                                                             pkHashConfigs,
+                                                                                             ",",
+                                                                                             ";"), ";");
+                                                                                     // schema.table
+                                                                                     for (String dynamicTopic : dynamicTopicArray) {
+                                                                                         DynamicTopicData data = new DynamicTopicData();
+
+                                                                                         if (!isWildCard(
+                                                                                             dynamicTopic)) {
+                                                                                             data.simpleName = dynamicTopic;
+                                                                                         } else {
+                                                                                             if (dynamicTopic
+                                                                                                 .contains("\\.")) {
+                                                                                                 data.tableRegexFilter = new AviaterRegexFilter(
+                                                                                                     dynamicTopic);
+                                                                                             } else {
+                                                                                                 data.schemaRegexFilter = new AviaterRegexFilter(
+                                                                                                     dynamicTopic);
+                                                                                             }
+                                                                                         }
+                                                                                         datas.add(data);
+                                                                                     }
+
+                                                                                     return datas;
+                                                                                 }
+                                                                             });
+
+    /**
+     * 按 schema 或者 schema+table 将 message 分配到对应topic
+     *
+     * @param message 原message
+     * @param defaultTopic 默认topic
+     * @param dynamicTopicConfigs 动态topic规则
+     * @return 分隔后的message map
+     */
+    public static Map<String, Message> messageTopics(Message message, String defaultTopic, String dynamicTopicConfigs) {
+        List<CanalEntry.Entry> entries;
+        if (message.isRaw()) {
+            List<ByteString> rawEntries = message.getRawEntries();
+            entries = new ArrayList<>(rawEntries.size());
+            for (ByteString byteString : rawEntries) {
+                CanalEntry.Entry entry;
+                try {
+                    entry = CanalEntry.Entry.parseFrom(byteString);
+                } catch (InvalidProtocolBufferException e) {
+                    throw new RuntimeException(e);
+                }
+                entries.add(entry);
+            }
+        } else {
+            entries = message.getEntries();
+        }
+        Map<String, Message> messages = new HashMap<>();
+        for (CanalEntry.Entry entry : entries) {
+            // 如果有topic路由,则忽略begin/end事件
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            String schemaName = entry.getHeader().getSchemaName();
+            String tableName = entry.getHeader().getTableName();
+
+            if (StringUtils.isEmpty(schemaName) || StringUtils.isEmpty(tableName)) {
+                put2MapMessage(messages, message.getId(), defaultTopic, entry);
+            } else {
+                Set<String> topics = matchTopics(schemaName + "." + tableName, dynamicTopicConfigs);
+                if (topics != null) {
+                    for (String topic : topics) {
+                        put2MapMessage(messages, message.getId(), topic, entry);
+                    }
+                } else {
+                    topics = matchTopics(schemaName, dynamicTopicConfigs);
+                    if (topics != null) {
+                        for (String topic : topics) {
+                            put2MapMessage(messages, message.getId(), topic, entry);
+                        }
+                    } else {
+                        put2MapMessage(messages, message.getId(), defaultTopic, entry);
+                    }
+                }
+            }
+        }
+        return messages;
+    }
+
+    /**
+     * 多线程构造message的rowChanged对象，比如为partition/flastMessage转化等处理 </br>
+     * 因为protobuf对象的序列化和反序列化是cpu密集型，串行执行会有代价
+     */
+    public static EntryRowData[] buildMessageData(Message message, ThreadPoolExecutor executor) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+        if (message.isRaw()) {
+            List<ByteString> rawEntries = message.getRawEntries();
+            final EntryRowData[] datas = new EntryRowData[rawEntries.size()];
+            int i = 0;
+            for (ByteString byteString : rawEntries) {
+                final int index = i;
+                template.submit(new Runnable() {
+
+                    @Override
+                    public void run() {
+                        try {
+                            Entry entry = Entry.parseFrom(byteString);
+                            CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+                            datas[index] = new EntryRowData();
+                            datas[index].entry = entry;
+                            datas[index].rowChange = rowChange;
+                        } catch (InvalidProtocolBufferException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                });
+
+                i++;
+            }
+
+            template.waitForResult();
+            return datas;
+        } else {
+            final EntryRowData[] datas = new EntryRowData[message.getEntries().size()];
+            int i = 0;
+            for (Entry entry : message.getEntries()) {
+                final int index = i;
+                template.submit(new Runnable() {
+
+                    @Override
+                    public void run() {
+                        try {
+                            CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+                            datas[index] = new EntryRowData();
+                            datas[index].entry = entry;
+                            datas[index].rowChange = rowChange;
+                        } catch (InvalidProtocolBufferException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                });
+
+                i++;
+            }
+
+            template.waitForResult();
+            return datas;
+        }
+    }
+
+    /**
+     * 将 message 分区
+     *
+     * @param partitionsNum 分区数
+     * @param pkHashConfigs 分区库表主键正则表达式
+     * @param databaseHash 是否取消根据database进行hash
+     * @return 分区message数组
+     */
+    @SuppressWarnings("unchecked")
+    public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs,
+                                             boolean databaseHash) {
+        if (partitionsNum == null) {
+            partitionsNum = 1;
+        }
+        Message[] partitionMessages = new Message[partitionsNum];
+        List<Entry>[] partitionEntries = new List[partitionsNum];
+        for (int i = 0; i < partitionsNum; i++) {
+            // 注意一下并发
+            partitionEntries[i] = Collections.synchronizedList(Lists.newArrayList());
+        }
+
+        for (EntryRowData data : datas) {
+            CanalEntry.Entry entry = data.entry;
+            CanalEntry.RowChange rowChange = data.rowChange;
+            // 如果有分区路由,则忽略begin/end事件
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            if (rowChange.getIsDdl()) {
+                partitionEntries[0].add(entry);
+            } else {
+                if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
+                    String database = entry.getHeader().getSchemaName();
+                    String table = entry.getHeader().getTableName();
+                    HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
+                    if (hashMode == null) {
+                        // 如果都没有匹配，发送到第一个分区
+                        partitionEntries[0].add(entry);
+                    } else if (hashMode.tableHash) {
+                        int hashCode = table.hashCode();
+                        int pkHash = Math.abs(hashCode) % partitionsNum;
+                        pkHash = Math.abs(pkHash);
+                        // tableHash not need split entry message
+                        partitionEntries[pkHash].add(entry);
+                    } else {
+                        // build new entry
+                        Entry.Builder builder = Entry.newBuilder(entry);
+                        RowChange.Builder rowChangeBuilder = RowChange.newBuilder(rowChange);
+
+                        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                            int hashCode = 0;
+                            if (databaseHash) {
+                                hashCode = database.hashCode();
+                            }
+                            CanalEntry.EventType eventType = rowChange.getEventType();
+                            List<CanalEntry.Column> columns = null;
+                            if (eventType == CanalEntry.EventType.DELETE) {
+                                columns = rowData.getBeforeColumnsList();
+                            } else {
+                                columns = rowData.getAfterColumnsList();
+                            }
+
+                            if (hashMode.autoPkHash) {
+                                // isEmpty use default pkNames
+                                for (CanalEntry.Column column : columns) {
+                                    if (column.getIsKey()) {
+                                        hashCode = hashCode ^ column.getValue().hashCode();
+                                    }
+                                }
+                            } else {
+                                for (CanalEntry.Column column : columns) {
+                                    if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
+                                        hashCode = hashCode ^ column.getValue().hashCode();
+                                    }
+                                }
+                            }
+
+                            int pkHash = Math.abs(hashCode) % partitionsNum;
+                            pkHash = Math.abs(pkHash);
+                            // clear rowDatas
+                            rowChangeBuilder.clearRowDatas();
+                            rowChangeBuilder.addRowDatas(rowData);
+                            builder.clearStoreValue();
+                            builder.setStoreValue(rowChangeBuilder.build().toByteString());
+                            partitionEntries[pkHash].add(builder.build());
+                        }
+                    }
+                } else {
+                    // 针对stmt/mixed binlog格式的query事件
+                    partitionEntries[0].add(entry);
+                }
+            }
+        }
+
+        for (int i = 0; i < partitionsNum; i++) {
+            List<Entry> entriesTmp = partitionEntries[i];
+            if (!entriesTmp.isEmpty()) {
+                partitionMessages[i] = new Message(id, entriesTmp);
+            }
+        }
+
+        return partitionMessages;
+    }
+
+    /**
+     * 将Message转换为FlatMessage
+     *
+     * @return FlatMessage列表
+     * @author agapple 2018年12月11日 下午1:28:32
+     */
+    public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
+        List<FlatMessage> flatMessages = new ArrayList<>();
+        for (EntryRowData entryRowData : datas) {
+            CanalEntry.Entry entry = entryRowData.entry;
+            CanalEntry.RowChange rowChange = entryRowData.rowChange;
+            // 如果有分区路由,则忽略begin/end事件
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            // build flatMessage
+            CanalEntry.EventType eventType = rowChange.getEventType();
+            FlatMessage flatMessage = new FlatMessage(id);
+            flatMessages.add(flatMessage);
+            flatMessage.setDatabase(entry.getHeader().getSchemaName());
+            flatMessage.setTable(entry.getHeader().getTableName());
+            flatMessage.setIsDdl(rowChange.getIsDdl());
+            flatMessage.setType(eventType.toString());
+            flatMessage.setEs(entry.getHeader().getExecuteTime());
+            flatMessage.setTs(System.currentTimeMillis());
+            flatMessage.setSql(rowChange.getSql());
+
+            if (!rowChange.getIsDdl()) {
+                Map<String, Integer> sqlType = new LinkedHashMap<>();
+                Map<String, String> mysqlType = new LinkedHashMap<>();
+                List<Map<String, String>> data = new ArrayList<>();
+                List<Map<String, String>> old = new ArrayList<>();
+
+                Set<String> updateSet = new HashSet<>();
+                boolean hasInitPkNames = false;
+                for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                    if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                        && eventType != CanalEntry.EventType.DELETE) {
+                        continue;
+                    }
+
+                    Map<String, String> row = new LinkedHashMap<>();
+                    List<CanalEntry.Column> columns;
+
+                    if (eventType == CanalEntry.EventType.DELETE) {
+                        columns = rowData.getBeforeColumnsList();
+                    } else {
+                        columns = rowData.getAfterColumnsList();
+                    }
+
+                    for (CanalEntry.Column column : columns) {
+                        if (!hasInitPkNames && column.getIsKey()) {
+                            flatMessage.addPkName(column.getName());
+                        }
+                        sqlType.put(column.getName(), column.getSqlType());
+                        mysqlType.put(column.getName(), column.getMysqlType());
+                        if (column.getIsNull()) {
+                            row.put(column.getName(), null);
+                        } else {
+                            row.put(column.getName(), column.getValue());
+                        }
+                        // 获取update为true的字段
+                        if (column.getUpdated()) {
+                            updateSet.add(column.getName());
+                        }
+                    }
+
+                    hasInitPkNames = true;
+                    if (!row.isEmpty()) {
+                        data.add(row);
+                    }
+
+                    if (eventType == CanalEntry.EventType.UPDATE) {
+                        Map<String, String> rowOld = new LinkedHashMap<>();
+                        for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                            if (updateSet.contains(column.getName())) {
+                                if (column.getIsNull()) {
+                                    rowOld.put(column.getName(), null);
+                                } else {
+                                    rowOld.put(column.getName(), column.getValue());
+                                }
+                            }
+                        }
+                        // update操作将记录修改前的值
+                        if (!rowOld.isEmpty()) {
+                            old.add(rowOld);
+                        }
+                    }
+                }
+                if (!sqlType.isEmpty()) {
+                    flatMessage.setSqlType(sqlType);
+                }
+                if (!mysqlType.isEmpty()) {
+                    flatMessage.setMysqlType(mysqlType);
+                }
+                if (!data.isEmpty()) {
+                    flatMessage.setData(data);
+                }
+                if (!old.isEmpty()) {
+                    flatMessage.setOld(old);
+                }
+            }
+        }
+        return flatMessages;
+    }
+
+    /**
+     * 将FlatMessage按指定的字段值hash拆分
+     *
+     * @param flatMessage flatMessage
+     * @param partitionsNum 分区数量
+     * @param pkHashConfigs hash映射
+     * @param databaseHash 是否取消根据database进行hash
+     * @return 拆分后的flatMessage数组
+     */
+    public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum, String pkHashConfigs,
+                                                 boolean databaseHash) {
+        if (partitionsNum == null) {
+            partitionsNum = 1;
+        }
+        FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
+
+        if (flatMessage.getIsDdl()) {
+            partitionMessages[0] = flatMessage;
+        } else {
+            if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
+                String database = flatMessage.getDatabase();
+                String table = flatMessage.getTable();
+                HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
+                if (hashMode == null) {
+                    // 如果都没有匹配，发送到第一个分区
+                    partitionMessages[0] = flatMessage;
+                } else if (hashMode.tableHash) {
+                    int hashCode = table.hashCode();
+                    int pkHash = Math.abs(hashCode) % partitionsNum;
+                    // math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
+                    pkHash = Math.abs(pkHash);
+                    partitionMessages[pkHash] = flatMessage;
+                } else {
+                    List<String> pkNames = hashMode.pkNames;
+                    if (hashMode.autoPkHash) {
+                        pkNames = flatMessage.getPkNames();
+                    }
+
+                    int idx = 0;
+                    for (Map<String, String> row : flatMessage.getData()) {
+                        int hashCode = 0;
+                        if (databaseHash) {
+                            hashCode = database.hashCode();
+                        }
+                        if (pkNames != null) {
+                            for (String pkName : pkNames) {
+                                String value = row.get(pkName);
+                                if (value == null) {
+                                    value = "";
+                                }
+                                hashCode = hashCode ^ value.hashCode();
+                            }
+                        }
+
+                        int pkHash = Math.abs(hashCode) % partitionsNum;
+                        // math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
+                        pkHash = Math.abs(pkHash);
+
+                        FlatMessage flatMessageTmp = partitionMessages[pkHash];
+                        if (flatMessageTmp == null) {
+                            flatMessageTmp = new FlatMessage(flatMessage.getId());
+                            partitionMessages[pkHash] = flatMessageTmp;
+                            flatMessageTmp.setDatabase(flatMessage.getDatabase());
+                            flatMessageTmp.setTable(flatMessage.getTable());
+                            flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
+                            flatMessageTmp.setType(flatMessage.getType());
+                            flatMessageTmp.setSql(flatMessage.getSql());
+                            flatMessageTmp.setSqlType(flatMessage.getSqlType());
+                            flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
+                            flatMessageTmp.setEs(flatMessage.getEs());
+                            flatMessageTmp.setTs(flatMessage.getTs());
+                            flatMessageTmp.setPkNames(flatMessage.getPkNames());
+                        }
+                        List<Map<String, String>> data = flatMessageTmp.getData();
+                        if (data == null) {
+                            data = new ArrayList<>();
+                            flatMessageTmp.setData(data);
+                        }
+                        data.add(row);
+                        if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
+                            List<Map<String, String>> old = flatMessageTmp.getOld();
+                            if (old == null) {
+                                old = new ArrayList<>();
+                                flatMessageTmp.setOld(old);
+                            }
+                            old.add(flatMessage.getOld().get(idx));
+                        }
+                        idx++;
+                    }
+                }
+            } else {
+                // 针对stmt/mixed binlog格式的query事件
+                partitionMessages[0] = flatMessage;
+            }
+        }
+        return partitionMessages;
+    }
+
+    /**
+     * match return List , not match return null
+     */
+    public static HashMode getPartitionHashColumns(String name, String pkHashConfigs) {
+        if (StringUtils.isEmpty(pkHashConfigs)) {
+            return null;
+        }
+
+        List<PartitionData> datas = partitionDatas.get(pkHashConfigs);
+        for (PartitionData data : datas) {
+            if (data.simpleName != null) {
+                if (data.simpleName.equalsIgnoreCase(name)) {
+                    return data.hashMode;
+                }
+            } else {
+                if (data.regexFilter.filter(name)) {
+                    return data.hashMode;
+                }
+            }
+        }
+
+        return null;
+    }
+
+    private static Set<String> matchTopics(String name, String dynamicTopicConfigs) {
+        String[] router = StringUtils.split(StringUtils.replace(dynamicTopicConfigs, ",", ";"), ";");
+        Set<String> topics = new HashSet<>();
+        for (String item : router) {
+            int i = item.indexOf(":");
+            if (i > -1) {
+                String topic = item.substring(0, i).trim();
+                String topicConfigs = item.substring(i + 1).trim();
+                if (matchDynamicTopic(name, topicConfigs)) {
+                    topics.add(topic);
+                    // 匹配了一个就退出
+                    break;
+                }
+            } else if (matchDynamicTopic(name, item)) {
+                // 匹配了一个就退出
+                topics.add(name.toLowerCase());
+                break;
+            }
+        }
+        return topics.isEmpty() ? null : topics;
+    }
+
+    public static boolean matchDynamicTopic(String name, String dynamicTopicConfigs) {
+        if (StringUtils.isEmpty(dynamicTopicConfigs)) {
+            return false;
+        }
+
+        boolean res = false;
+        List<DynamicTopicData> datas = dynamicTopicDatas.get(dynamicTopicConfigs);
+        for (DynamicTopicData data : datas) {
+            if (data.simpleName != null) {
+                if (data.simpleName.equalsIgnoreCase(name)) {
+                    res = true;
+                    break;
+                }
+            } else if (name.contains(".")) {
+                if (data.tableRegexFilter != null && data.tableRegexFilter.filter(name)) {
+                    res = true;
+                    break;
+                }
+            } else {
+                if (data.schemaRegexFilter != null && data.schemaRegexFilter.filter(name)) {
+                    res = true;
+                    break;
+                }
+            }
+        }
+        return res;
+    }
+
+    public static boolean checkPkNamesHasContain(List<String> pkNames, String name) {
+        for (String pkName : pkNames) {
+            if (pkName.equalsIgnoreCase(name)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private static boolean isWildCard(String value) {
+        // not contaiins '.' ?
+        return StringUtils.containsAny(value,
+            new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' });
+    }
+
+    private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,
+                                       CanalEntry.Entry entry) {
+        Message message = messageMap.get(topicName);
+        if (message == null) {
+            message = new Message(messageId, new ArrayList<CanalEntry.Entry>());
+            messageMap.put(topicName, message);
+        }
+        message.getEntries().add(entry);
+    }
+
+    public static class PartitionData {
+
+        public String             simpleName;
+        public AviaterRegexFilter regexFilter;
+        public HashMode           hashMode = new HashMode();
+    }
+
+    public static class HashMode {
+
+        public boolean      autoPkHash = false;
+        public boolean      tableHash  = false;
+        public List<String> pkNames    = Lists.newArrayList();
+    }
+
+    public static class DynamicTopicData {
+
+        public String             simpleName;
+        public AviaterRegexFilter schemaRegexFilter;
+        public AviaterRegexFilter tableRegexFilter;
+    }
+
+    public static class EntryRowData {
+
+        public Entry     entry;
+        public RowChange rowChange;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java

@@ -0,0 +1,43 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.util.Properties;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * MQ producer SPI 接口
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public interface CanalMQProducer {
+
+    /**
+     * Init producer.
+     */
+    void init(Properties properties);
+
+    /**
+     * Get base mq properties
+     * 
+     * @return MQProperties
+     */
+    MQProperties getMqProperties();
+
+    /**
+     * Send canal message to related topic
+     *
+     * @param canalDestination canal mq destination
+     * @param message canal message
+     */
+    void send(MQDestination canalDestination, Message message, Callback callback);
+
+    /**
+     * Stop MQ producer service
+     */
+    void stop();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java

@@ -0,0 +1,55 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+
+/**
+ * Canal/MQ consumer SPI 接口
+ *
+ * @author rewerma @ 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public interface CanalMsgConsumer {
+
+    /**
+     * 初始化
+     * 
+     * @param properties consumer properties
+     * @param topic topic/destination
+     * @param groupId mq group id
+     */
+    void init(Properties properties, String topic, String groupId);
+
+    /**
+     * 连接Canal/MQ
+     */
+    void connect();
+
+    /**
+     * 批量拉取数据
+     * 
+     * @param timeout 超时时间
+     * @param unit 时间单位
+     * @return Message列表
+     */
+    List<CommonMessage> getMessage(Long timeout, TimeUnit unit);
+
+    /**
+     * 提交
+     */
+    void ack();
+
+    /**
+     * 回滚
+     */
+    void rollback();
+
+    /**
+     * 断开连接
+     */
+    void disconnect();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java

@@ -0,0 +1,435 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SPI 类加载器
+ *
+ * @author rewerma 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class ExtensionLoader<T> {
+
+    private static final Logger                                      logger                     = LoggerFactory
+        .getLogger(ExtensionLoader.class);
+
+    private static final String                                      SERVICES_DIRECTORY         = "META-INF/services/";
+
+    private static final String                                      CANAL_DIRECTORY            = "META-INF/canal/";
+
+    private static final String                                      DEFAULT_CLASSLOADER_POLICY = "internal";
+
+    private static final Pattern                                     NAME_SEPARATOR             = Pattern
+        .compile("\\s*[,]+\\s*");
+
+    private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS          = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<Class<?>, Object>             EXTENSION_INSTANCES        = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<String, Object>               EXTENSION_KEY_INSTANCE     = new ConcurrentHashMap<>();
+
+    private final Class<?>                                           type;
+
+    private final String                                             classLoaderPolicy;
+
+    private final ConcurrentMap<Class<?>, String>                    cachedNames                = new ConcurrentHashMap<>();
+
+    private final Holder<Map<String, Class<?>>>                      cachedClasses              = new Holder<>();
+
+    private final ConcurrentMap<String, Holder<Object>>              cachedInstances            = new ConcurrentHashMap<>();
+
+    private String                                                   cachedDefaultName;
+
+    private ConcurrentHashMap<String, IllegalStateException>         exceptions                 = new ConcurrentHashMap<>();
+
+    private static <T> boolean withExtensionAnnotation(Class<T> type) {
+        return type.isAnnotationPresent(SPI.class);
+    }
+
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
+        return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
+        if (type == null) throw new IllegalArgumentException("Extension type == null");
+        if (!type.isInterface()) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
+        }
+        if (!withExtensionAnnotation(type)) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+                                               + SPI.class.getSimpleName() + " Annotation!");
+        }
+
+        ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        if (loader == null) {
+            EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
+            loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        }
+        return loader;
+    }
+
+    public ExtensionLoader(Class<?> type){
+        this.type = type;
+        this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY;
+    }
+
+    public ExtensionLoader(Class<?> type, String classLoaderPolicy){
+        this.type = type;
+        this.classLoaderPolicy = classLoaderPolicy;
+    }
+
+    /**
+     * 返回指定名字的扩展
+     *
+     * @param name
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name, String spiDir, String standbyDir) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension(spiDir, standbyDir);
+        }
+        Holder<Object> holder = cachedInstances.get(name);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(name, new Holder<>());
+            holder = cachedInstances.get(name);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name, spiDir, standbyDir);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name, String key, String spiDir, String standbyDir) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension(spiDir, standbyDir);
+        }
+        String extKey = name + "-" + StringUtils.trimToEmpty(key);
+        Holder<Object> holder = cachedInstances.get(extKey);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(extKey, new Holder<>());
+            holder = cachedInstances.get(extKey);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name, key, spiDir, standbyDir);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    /**
+     * 返回缺省的扩展，如果没有设置则返回<code>null</code>
+     */
+    public T getDefaultExtension(String spiDir, String standbyDir) {
+        getExtensionClasses(spiDir, standbyDir);
+        if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
+            return null;
+        }
+        return getExtension(cachedDefaultName, spiDir, standbyDir);
+    }
+
+    @SuppressWarnings("unchecked")
+    public T createExtension(String name, String spiDir, String standbyDir) {
+        Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_INSTANCES.get(clazz);
+            if (instance == null) {
+                EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
+                instance = (T) EXTENSION_INSTANCES.get(clazz);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(),
+                t);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private T createExtension(String name, String key, String spiDir, String standbyDir) {
+        Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
+            if (instance == null) {
+                EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance());
+                instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(),
+                t);
+        }
+    }
+
+    private Map<String, Class<?>> getExtensionClasses(String spiDir, String standbyDir) {
+        Map<String, Class<?>> classes = cachedClasses.get();
+        if (classes == null) {
+            synchronized (cachedClasses) {
+                classes = cachedClasses.get();
+                if (classes == null) {
+                    classes = loadExtensionClasses(spiDir, standbyDir);
+                    cachedClasses.set(classes);
+                }
+            }
+        }
+
+        return classes;
+    }
+
+    private String getJarDirectoryPath() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("");
+        String dirtyPath;
+        if (url != null) {
+            dirtyPath = url.toString();
+        } else {
+            File file = new File("");
+            dirtyPath = file.getAbsolutePath();
+        }
+        String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
+                                                                // file:/ and
+                                                                // everything
+                                                                // before it
+        jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything
+                                                       // after .jar, if .jar
+                                                       // exists in dirtyPath
+        jarPath = jarPath.replaceAll("%20", " "); // necessary if path has
+                                                  // spaces within
+        if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run
+                                         // the app using Spring Tools Suit play
+                                         // button.
+            jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
+        }
+        Path path = Paths.get(jarPath).getParent(); // Paths - from java 8
+        if (path != null) {
+            return path.toString();
+        }
+        return null;
+    }
+
+    private Map<String, Class<?>> loadExtensionClasses(String spiDir, String standbyDir) {
+        final SPI defaultAnnotation = type.getAnnotation(SPI.class);
+        if (defaultAnnotation != null) {
+            String value = defaultAnnotation.value();
+            if ((value = value.trim()).length() > 0) {
+                String[] names = NAME_SEPARATOR.split(value);
+                if (names.length > 1) {
+                    throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+                                                    + ": " + Arrays.toString(names));
+                }
+                if (names.length == 1) cachedDefaultName = names[0];
+            }
+        }
+
+        Map<String, Class<?>> extensionClasses = new HashMap<>();
+
+        if (spiDir != null && standbyDir != null) {
+            // 1. plugin folder，customized extension classLoader （jar_dir/plugin）
+            String dir = File.separator + this.getJarDirectoryPath() + spiDir; // + "plugin";
+
+            File externalLibDir = new File(dir);
+            if (!externalLibDir.exists()) {
+                externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir);
+            }
+            logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath());
+            if (externalLibDir.exists()) {
+                File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar"));
+                if (files != null) {
+                    for (File f : files) {
+                        URL url;
+                        try {
+                            url = f.toURI().toURL();
+                        } catch (MalformedURLException e) {
+                            throw new RuntimeException("load extension jar failed!", e);
+                        }
+
+                        ClassLoader parent = Thread.currentThread().getContextClassLoader();
+                        URLClassLoader localClassLoader;
+                        if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
+                            || DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
+                            localClassLoader = new URLClassExtensionLoader(new URL[] { url });
+                        } else {
+                            localClassLoader = new URLClassLoader(new URL[] { url }, parent);
+                        }
+
+                        loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
+                        loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
+                    }
+                }
+            }
+        }
+
+        // 2. load inner extension class with default classLoader
+        ClassLoader classLoader = findClassLoader();
+        loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
+        loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
+
+        return extensionClasses;
+    }
+
+    private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
+        String fileName = dir + type.getName();
+        try {
+            Enumeration<URL> urls;
+            if (classLoader != null) {
+                urls = classLoader.getResources(fileName);
+            } else {
+                urls = ClassLoader.getSystemResources(fileName);
+            }
+            if (urls != null) {
+                while (urls.hasMoreElements()) {
+                    URL url = urls.nextElement();
+                    try {
+                        BufferedReader reader = null;
+                        try {
+                            reader = new BufferedReader(
+                                new InputStreamReader(url.openStream(), StandardCharsets.UTF_8));
+                            String line = null;
+                            while ((line = reader.readLine()) != null) {
+                                final int ci = line.indexOf('#');
+                                if (ci >= 0) line = line.substring(0, ci);
+                                line = line.trim();
+                                if (line.length() > 0) {
+                                    try {
+                                        String name = null;
+                                        int i = line.indexOf('=');
+                                        if (i > 0) {
+                                            name = line.substring(0, i).trim();
+                                            line = line.substring(i + 1).trim();
+                                        }
+                                        if (line.length() > 0) {
+                                            Class<?> clazz = classLoader.loadClass(line);
+                                            // Class<?> clazz =
+                                            // Class.forName(line, true,
+                                            // classLoader);
+                                            if (!type.isAssignableFrom(clazz)) {
+                                                throw new IllegalStateException(
+                                                    "Error when load extension class(interface: " + type
+                                                                                + ", class line: " + clazz.getName()
+                                                                                + "), class " + clazz.getName()
+                                                                                + "is not subtype of interface.");
+                                            } else {
+                                                try {
+                                                    clazz.getConstructor(type);
+                                                } catch (NoSuchMethodException e) {
+                                                    clazz.getConstructor();
+                                                    String[] names = NAME_SEPARATOR.split(name);
+                                                    if (names != null && names.length > 0) {
+                                                        for (String n : names) {
+                                                            if (!cachedNames.containsKey(clazz)) {
+                                                                cachedNames.put(clazz, n);
+                                                            }
+                                                            Class<?> c = extensionClasses.get(n);
+                                                            if (c == null) {
+                                                                extensionClasses.put(n, clazz);
+                                                            } else if (c != clazz) {
+                                                                cachedNames.remove(clazz);
+                                                                throw new IllegalStateException(
+                                                                    "Duplicate extension " + type.getName() + " name "
+                                                                                                + n + " on "
+                                                                                                + c.getName() + " and "
+                                                                                                + clazz.getName());
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    } catch (Throwable t) {
+                                        IllegalStateException e = new IllegalStateException(
+                                            "Failed to load extension class(interface: " + type + ", class line: "
+                                                                                            + line + ") in " + url
+                                                                                            + ", cause: "
+                                                                                            + t.getMessage(),
+                                            t);
+                                        exceptions.put(line, e);
+                                    }
+                                }
+                            } // end of while read lines
+                        } finally {
+                            if (reader != null) {
+                                reader.close();
+                            }
+                        }
+                    } catch (Throwable t) {
+                        logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+                                     + ") in " + url,
+                            t);
+                    }
+                } // end of while urls
+            }
+        } catch (Throwable t) {
+            logger.error(
+                "Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
+                t);
+        }
+    }
+
+    @SuppressWarnings("unused")
+    private static ClassLoader findClassLoader() {
+        return ExtensionLoader.class.getClassLoader();
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getName() + "[" + type.getName() + "]";
+    }
+
+    private static class Holder<T> {
+
+        private volatile T value;
+
+        private void set(T value) {
+            this.value = value;
+        }
+
+        private T get() {
+            return value;
+        }
+
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java

@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.lang.annotation.*;
+
+/**
+ * SPI装载器注解
+ *
+ * @author rewerma @ 2018-10-20
+ * @version 1.0.0
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface SPI {
+
+    // Default SPI name
+    String value() default "";
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java

@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Enumeration;
+import java.util.NoSuchElementException;
+
+public class URLClassExtensionLoader extends URLClassLoader {
+    public URLClassExtensionLoader(URL[] urls) {
+        super(urls);
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        Class<?> c = findLoadedClass(name);
+        if (c != null) {
+            return c;
+        }
+
+        if (name.startsWith("java.") || name.startsWith("org.slf4j.")
+                || name.startsWith("org.apache.logging")
+                || name.startsWith("org.apache.commons.logging.")) {
+            // || name.startsWith("org.apache.hadoop."))
+            // {
+            c = super.loadClass(name);
+        }
+        if (c != null) return c;
+
+        try {
+            // 先加载jar内的class，可避免jar冲突
+            c = findClass(name);
+        } catch (ClassNotFoundException e) {
+            c = null;
+        }
+        if (c != null) {
+            return c;
+        }
+
+        return super.loadClass(name);
+    }
+
+    @Override
+    public Enumeration<URL> getResources(String name) throws IOException {
+        @SuppressWarnings("unchecked")
+        Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
+
+        tmp[0] = findResources(name); // local class
+        // path first
+        // tmp[1] = super.getResources(name);
+
+        return new CompoundEnumeration<>(tmp);
+    }
+
+    private static class CompoundEnumeration<E> implements Enumeration<E> {
+
+        private Enumeration<E>[] enums;
+        private int              index = 0;
+
+        public CompoundEnumeration(Enumeration<E>[] enums){
+            this.enums = enums;
+        }
+
+        private boolean next() {
+            while (this.index < this.enums.length) {
+                if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
+                    return true;
+                }
+
+                ++this.index;
+            }
+
+            return false;
+        }
+
+        public boolean hasMoreElements() {
+            return this.next();
+        }
+
+        public E nextElement() {
+            if (!this.next()) {
+                throw new NoSuchElementException();
+            } else {
+                return this.enums[this.index].nextElement();
+            }
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java

@@ -0,0 +1,13 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+/**
+ * MQ 回调类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public interface Callback {
+    void commit();
+
+    void rollback();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java

@@ -0,0 +1,132 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.util.List;
+
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.protocol.Message;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
+
+/**
+ * Canal message 序列化工具类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class CanalMessageSerializerUtil {
+
+    @SuppressWarnings("deprecation")
+    public static byte[] serializer(Message data, boolean filterTransactionEntry) {
+        try {
+            if (data != null) {
+                if (data.getId() != -1) {
+                    if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
+                        // for performance
+                        List<ByteString> rowEntries = data.getRawEntries();
+                        // message size
+                        int messageSize = 0;
+                        messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
+
+                        int dataSize = 0;
+                        for (int i = 0; i < rowEntries.size(); i++) {
+                            dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
+                        }
+                        messageSize += dataSize;
+                        messageSize += 1 * rowEntries.size();
+                        // packet size
+                        int size = 0;
+                        size += CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
+                        size += CodedOutputStream.computeTagSize(5)
+                                + CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
+                        // build data
+                        byte[] body = new byte[size];
+                        CodedOutputStream output = CodedOutputStream.newInstance(body);
+                        output.writeEnum(3, PacketType.MESSAGES.getNumber());
+
+                        output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+                        output.writeRawVarint32(messageSize);
+                        // message
+                        output.writeInt64(1, data.getId());
+                        for (int i = 0; i < rowEntries.size(); i++) {
+                            output.writeBytes(2, rowEntries.get(i));
+                        }
+                        output.checkNoSpaceLeft();
+                        return body;
+                    } else if (!CollectionUtils.isEmpty(data.getEntries())) {
+                        // mq模式只会走到非rowEntry模式
+                        CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
+                        for (CanalEntry.Entry entry : data.getEntries()) {
+                            if (filterTransactionEntry
+                                && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
+                                continue;
+                            }
+
+                            messageBuilder.addMessages(entry.toByteString());
+                        }
+
+                        CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
+                        packetBuilder.setType(PacketType.MESSAGES);
+                        packetBuilder.setVersion(1);
+                        packetBuilder.setBody(messageBuilder.build().toByteString());
+                        return packetBuilder.build().toByteArray();
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Error when serializing message to byte[] ");
+        }
+        return null;
+    }
+
+    public static Message deserializer(byte[] data) {
+        return deserializer(data, false);
+    }
+
+    public static Message deserializer(byte[] data, boolean lazyParseEntry) {
+        try {
+            if (data == null) {
+                return null;
+            } else {
+                CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
+                switch (p.getType()) {
+                    case MESSAGES: {
+                        if (!p.getCompression().equals(CanalPacket.Compression.NONE)
+                                && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
+                            throw new CanalClientException("compression is not supported in this connector");
+                        }
+
+                        CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
+                        Message result = new Message(messages.getBatchId());
+                        if (lazyParseEntry) {
+                            // byteString
+                            result.setRawEntries(messages.getMessagesList());
+                            result.setRaw(true);
+                        } else {
+                            for (ByteString byteString : messages.getMessagesList()) {
+                                result.addEntry(CanalEntry.Entry.parseFrom(byteString));
+                            }
+                            result.setRaw(false);
+                        }
+                        return result;
+                    }
+                    case ACK: {
+                        CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
+                        throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
+                    }
+                    default: {
+                        throw new CanalClientException("unexpected packet type: " + p.getType());
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new CanalClientException("deserializer failed", e);
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java

@@ -0,0 +1,46 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.util.Date;
+
+public class DateUtil {
+    private static DateTimeZone dateTimeZone;
+
+    static {
+        dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE);
+    }
+
+    /**
+     * 通用日期时间字符解析
+     *
+     * @param datetimeStr 日期时间字符串
+     * @return Date
+     */
+    public static Date parseDate(String datetimeStr) {
+        if (StringUtils.isEmpty(datetimeStr)) {
+            return null;
+        }
+        datetimeStr = datetimeStr.trim().replace('/', '-');
+        if (datetimeStr.contains("-")) {
+            if (datetimeStr.contains(":")) {
+                datetimeStr = datetimeStr.replace(" ", "T");
+            }
+        } else if (datetimeStr.contains(":")) {
+            datetimeStr = "T" + datetimeStr;
+        } else {
+            if (datetimeStr.length() == 8) {
+                String year = datetimeStr.substring(0, 4);
+                String month = datetimeStr.substring(4, 6);
+                String day = datetimeStr.substring(6, 8);
+                datetimeStr = year + "-" + month + "-" + day;
+            }
+        }
+
+        DateTime dateTime = new DateTime(datetimeStr, dateTimeZone);
+
+        return dateTime.toDate();
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java

@@ -0,0 +1,165 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 类型转换工具类
+ *
+ * @author rewerma 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class JdbcTypeUtil {
+
+    private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
+
+    public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
+        if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
+            return rs.getByte(columnName);
+        } else {
+            return rs.getObject(columnName);
+        }
+    }
+
+    public static Class<?> jdbcType2javaType(int jdbcType) {
+        switch (jdbcType) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                // return Boolean.class;
+            case Types.TINYINT:
+                return Byte.TYPE;
+            case Types.SMALLINT:
+                return Short.class;
+            case Types.INTEGER:
+                return Integer.class;
+            case Types.BIGINT:
+                return Long.class;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+                return BigDecimal.class;
+            case Types.REAL:
+                return Float.class;
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                return Double.class;
+            case Types.CHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+                return String.class;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                return byte[].class;
+            case Types.DATE:
+                return Date.class;
+            case Types.TIME:
+                return Time.class;
+            case Types.TIMESTAMP:
+                return Timestamp.class;
+            default:
+                return String.class;
+        }
+    }
+
+    private static boolean isText(String columnType) {
+        return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
+               || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
+    }
+
+    public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
+        if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR
+                                                    || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
+            return null;
+        }
+
+        try {
+            Object res;
+            switch (sqlType) {
+                case Types.INTEGER:
+                    res = Integer.parseInt(value);
+                    break;
+                case Types.SMALLINT:
+                    res = Short.parseShort(value);
+                    break;
+                case Types.BIT:
+                case Types.TINYINT:
+                    res = Byte.parseByte(value);
+                    break;
+                case Types.BIGINT:
+                    if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
+                        res = new BigInteger(value);
+                    } else {
+                        res = Long.parseLong(value);
+                    }
+                    break;
+                // case Types.BIT:
+                case Types.BOOLEAN:
+                    res = !"0".equals(value);
+                    break;
+                case Types.DOUBLE:
+                case Types.FLOAT:
+                    res = Double.parseDouble(value);
+                    break;
+                case Types.REAL:
+                    res = Float.parseFloat(value);
+                    break;
+                case Types.DECIMAL:
+                case Types.NUMERIC:
+                    res = new BigDecimal(value);
+                    break;
+                case Types.BINARY:
+                case Types.VARBINARY:
+                case Types.LONGVARBINARY:
+                case Types.BLOB:
+                    res = value.getBytes("ISO-8859-1");
+                    break;
+                case Types.DATE:
+                    if (!value.startsWith("0000-00-00")) {
+                        java.util.Date date = DateUtil.parseDate(value);
+                        if (date != null) {
+                            res = new Date(date.getTime());
+                        } else {
+                            res = null;
+                        }
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.TIME: {
+                    java.util.Date date = DateUtil.parseDate(value);
+                    if (date != null) {
+                        res = new Time(date.getTime());
+                    } else {
+                        res = null;
+                    }
+                    break;
+                }
+                case Types.TIMESTAMP:
+                    if (!value.startsWith("0000-00-00")) {
+                        java.util.Date date = DateUtil.parseDate(value);
+                        if (date != null) {
+                            res = new Timestamp(date.getTime());
+                        } else {
+                            res = null;
+                        }
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.CLOB:
+                default:
+                    res = value;
+                    break;
+            }
+            return res;
+        } catch (Exception e) {
+            logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
+            return value;
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java

@@ -0,0 +1,131 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.util.*;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Message对象解析工具类
+ *
+ * @author rewerma 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class MessageUtil {
+
+    public static List<CommonMessage> convert(Message message) {
+        if (message == null) {
+            return null;
+        }
+        List<CanalEntry.Entry> entries = message.getEntries();
+        List<CommonMessage> msgs = new ArrayList<>(entries.size());
+        for (CanalEntry.Entry entry : entries) {
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            CanalEntry.RowChange rowChange;
+            try {
+                rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
+                        e);
+            }
+
+            CanalEntry.EventType eventType = rowChange.getEventType();
+
+            final CommonMessage msg = new CommonMessage();
+            msg.setIsDdl(rowChange.getIsDdl());
+            msg.setDatabase(entry.getHeader().getSchemaName());
+            msg.setTable(entry.getHeader().getTableName());
+            msg.setType(eventType.toString());
+            msg.setEs(entry.getHeader().getExecuteTime());
+            msg.setIsDdl(rowChange.getIsDdl());
+            msg.setTs(System.currentTimeMillis());
+            msg.setSql(rowChange.getSql());
+            msgs.add(msg);
+            List<Map<String, Object>> data = new ArrayList<>();
+            List<Map<String, Object>> old = new ArrayList<>();
+
+            if (!rowChange.getIsDdl()) {
+                Set<String> updateSet = new HashSet<>();
+                msg.setPkNames(new ArrayList<>());
+                int i = 0;
+                for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                    if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                            && eventType != CanalEntry.EventType.DELETE) {
+                        continue;
+                    }
+
+                    Map<String, Object> row = new LinkedHashMap<>();
+                    List<CanalEntry.Column> columns;
+
+                    if (eventType == CanalEntry.EventType.DELETE) {
+                        columns = rowData.getBeforeColumnsList();
+                    } else {
+                        columns = rowData.getAfterColumnsList();
+                    }
+
+                    for (CanalEntry.Column column : columns) {
+                        if (i == 0) {
+                            if (column.getIsKey()) {
+                                msg.getPkNames().add(column.getName());
+                            }
+                        }
+                        if (column.getIsNull()) {
+                            row.put(column.getName(), null);
+                        } else {
+                            row.put(column.getName(),
+                                    JdbcTypeUtil.typeConvert(msg.getTable(),
+                                            column.getName(),
+                                            column.getValue(),
+                                            column.getSqlType(),
+                                            column.getMysqlType()));
+                        }
+                        // 获取update为true的字段
+                        if (column.getUpdated()) {
+                            updateSet.add(column.getName());
+                        }
+                    }
+                    if (!row.isEmpty()) {
+                        data.add(row);
+                    }
+
+                    if (eventType == CanalEntry.EventType.UPDATE) {
+                        Map<String, Object> rowOld = new LinkedHashMap<>();
+                        for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                            if (updateSet.contains(column.getName())) {
+                                if (column.getIsNull()) {
+                                    rowOld.put(column.getName(), null);
+                                } else {
+                                    rowOld.put(column.getName(),
+                                            JdbcTypeUtil.typeConvert(msg.getTable(),
+                                                    column.getName(),
+                                                    column.getValue(),
+                                                    column.getSqlType(),
+                                                    column.getMysqlType()));
+                                }
+                            }
+                        }
+                        // update操作将记录修改前的值
+                        if (!rowOld.isEmpty()) {
+                            old.add(rowOld);
+                        }
+                    }
+
+                    i++;
+                }
+                if (!data.isEmpty()) {
+                    msg.setData(data);
+                }
+                if (!old.isEmpty()) {
+                    msg.setOld(old);
+                }
+            }
+        }
+
+        return msgs;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java

@@ -0,0 +1,22 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+
+public class TimeZone {
+    public final static String LOCATION_TIME_ZONE;    // 当前时区
+
+    static {
+        java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault();
+        int rawOffset = localTimeZone.getRawOffset();
+        String symbol = "+";
+        if (rawOffset < 0) {
+            symbol = "-";
+        }
+        rawOffset = Math.abs(rawOffset);
+        int offsetHour = rawOffset / 3600000;
+        int offsetMinute = rawOffset % 3600000 / 60000;
+        String hour = String.format("%1$02d", offsetHour);
+        String minute = String.format("%1$02d", offsetMinute);
+        LOCATION_TIME_ZONE = symbol + hour + ":" + minute;
+        java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("GMT" + LOCATION_TIME_ZONE));
+    }
+}

connector/kafka-connector/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.kafka</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector kafka module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka_2.11</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java

@@ -0,0 +1,16 @@
+package com.alibaba.otter.canal.connector.kafka.config;
+
+/**
+ * Kafka 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class KafkaConstants {
+
+    public static final String ROOT                              = "canal";
+
+    public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE    = ROOT + "." + "mq.kafka.kerberos.enable";
+    public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE = ROOT + "." + "mq.kafka.kerberos.krb5.file";
+    public static final String CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE = ROOT + "." + "mq.kafka.kerberos.jaas.file";
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java

@@ -0,0 +1,53 @@
+package com.alibaba.otter.canal.connector.kafka.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * Kafka producer 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class KafkaProducerConfig extends MQProperties {
+
+    private Map<String, Object> kafkaProperties = new LinkedHashMap<>();
+
+    private boolean             kerberosEnabled         = false;
+    private String              krb5File;
+    private String              jaasFile;
+
+    public Map<String, Object> getKafkaProperties() {
+        return kafkaProperties;
+    }
+
+    public void setKafkaProperties(Map<String, Object> kafkaProperties) {
+        this.kafkaProperties = kafkaProperties;
+    }
+
+    public boolean isKerberosEnabled() {
+        return kerberosEnabled;
+    }
+
+    public void setKerberosEnabled(boolean kerberosEnabled) {
+        this.kerberosEnabled = kerberosEnabled;
+    }
+
+    public String getKrb5File() {
+        return krb5File;
+    }
+
+    public void setKrb5File(String krb5File) {
+        this.krb5File = krb5File;
+    }
+
+    public String getJaasFile() {
+        return jaasFile;
+    }
+
+    public void setJaasFile(String jaasFile) {
+        this.jaasFile = jaasFile;
+    }
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java

@@ -0,0 +1,137 @@
+package com.alibaba.otter.canal.connector.kafka.consumer;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.StringDeserializer;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Kafka consumer SPI 实现
+ *
+ * @author rewerma @ 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public class CanalKafkaConsumer implements CanalMsgConsumer {
+
+    private static final String      PREFIX_KAFKA_CONFIG = "kafka.";
+
+    private KafkaConsumer<String, ?> kafkaConsumer;
+    private boolean                  flatMessage         = true;
+    private String                   topic;
+
+    private Map<Integer, Long>       currentOffsets      = new ConcurrentHashMap<>();
+    private Properties               kafkaProperties     = new Properties();
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        this.topic = topic;
+
+        Boolean flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        if (flatMessage != null) {
+            this.flatMessage = flatMessage;
+        }
+        for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+            String k = (String) entry.getKey();
+            Object v = entry.getValue();
+            if (k.startsWith(PREFIX_KAFKA_CONFIG) && v != null) {
+                kafkaProperties.put(k.substring(PREFIX_KAFKA_CONFIG.length()), v);
+            }
+        }
+        kafkaProperties.put("group.id", groupId);
+        kafkaProperties.put("key.deserializer", StringDeserializer.class);
+        kafkaProperties.put("client.id", UUID.randomUUID().toString().substring(0, 6));
+    }
+
+    @Override
+    public void connect() {
+        if (this.flatMessage) {
+            kafkaProperties.put("value.deserializer", StringDeserializer.class);
+            this.kafkaConsumer = new KafkaConsumer<String, String>(kafkaProperties);
+        } else {
+            kafkaProperties.put("value.deserializer", KafkaMessageDeserializer.class);
+            this.kafkaConsumer = new KafkaConsumer<String, Message>(kafkaProperties);
+        }
+        kafkaConsumer.subscribe(Collections.singletonList(topic));
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        if (!flatMessage) {
+            ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer
+                .poll(unit.toMillis(timeout));
+            if (!records.isEmpty()) {
+                currentOffsets.clear();
+                List<CommonMessage> messages = new ArrayList<>();
+                for (ConsumerRecord<String, Message> record : records) {
+                    if (currentOffsets.get(record.partition()) == null) {
+                        currentOffsets.put(record.partition(), record.offset());
+                    }
+                    messages.addAll(MessageUtil.convert(record.value()));
+                }
+                return messages;
+            }
+        } else {
+            ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer
+                .poll(unit.toMillis(timeout));
+
+            if (!records.isEmpty()) {
+                List<CommonMessage> messages = new ArrayList<>();
+                currentOffsets.clear();
+                for (ConsumerRecord<String, String> record : records) {
+                    if (currentOffsets.get(record.partition()) == null) {
+                        currentOffsets.put(record.partition(), record.offset());
+                    }
+                    String flatMessageJson = record.value();
+                    CommonMessage flatMessages = JSON.parseObject(flatMessageJson, CommonMessage.class);
+                    messages.add(flatMessages);
+                }
+                return messages;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        // 回滚所有分区
+        if (kafkaConsumer != null) {
+            for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
+                kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey()));
+                kafkaConsumer.commitSync();
+            }
+        }
+    }
+
+    @Override
+    public void ack() {
+        if (kafkaConsumer != null) {
+            kafkaConsumer.commitSync();
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        if (kafkaConsumer != null) {
+            kafkaConsumer.unsubscribe();
+        }
+        if (kafkaConsumer != null) {
+            kafkaConsumer.close();
+            kafkaConsumer = null;
+        }
+    }
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.connector.kafka.consumer;
+
+import java.util.Map;
+
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import org.apache.kafka.common.serialization.Deserializer;
+
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Kafka Message类的反序列化
+ *
+ * @author rewerma @ 2018-6-12
+ * @version 1.0.0
+ */
+public class KafkaMessageDeserializer implements Deserializer<Message> {
+
+    @Override
+    public void configure(Map<String, ?> configs, boolean isKey) {
+    }
+
+    @Override
+    public Message deserialize(String topic1, byte[] data) {
+        return CanalMessageSerializerUtil.deserializer(data);
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java

@@ -0,0 +1,254 @@
+package com.alibaba.otter.canal.connector.kafka.producer;
+
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * kafka producer SPI 实现
+ *
+ * @author rewerma 2018-6-11 下午05:30:49
+ * @version 1.0.0
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+@SPI("kafka")
+public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger      logger              = LoggerFactory.getLogger(CanalKafkaProducer.class);
+
+    private static final String      PREFIX_KAFKA_CONFIG = "kafka.";
+
+    private Producer<String, byte[]> producer;
+
+    @Override
+    public void init(Properties properties) {
+        KafkaProducerConfig kafkaProducerConfig = new KafkaProducerConfig();
+        this.mqProperties = kafkaProducerConfig;
+        super.init(properties);
+        // load properties
+        this.loadKafkaProperties(properties);
+
+        Properties kafkaProperties = new Properties();
+        kafkaProperties.putAll(kafkaProducerConfig.getKafkaProperties());
+        kafkaProperties.put("key.serializer", StringSerializer.class);
+        if (kafkaProducerConfig.isKerberosEnabled()) {
+            File krb5File = new File(kafkaProducerConfig.getKrb5File());
+            File jaasFile = new File(kafkaProducerConfig.getJaasFile());
+            if (krb5File.exists() && jaasFile.exists()) {
+                // 配置kerberos认证，需要使用绝对路径
+                System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath());
+                System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath());
+                System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
+                kafkaProperties.put("security.protocol", "SASL_PLAINTEXT");
+                kafkaProperties.put("sasl.kerberos.service.name", "kafka");
+            } else {
+                String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it";
+                logger.error(errorMsg);
+                throw new RuntimeException(errorMsg);
+            }
+        }
+        kafkaProperties.put("value.serializer", KafkaMessageSerializer.class);
+        producer = new KafkaProducer<>(kafkaProperties);
+    }
+
+    private void loadKafkaProperties(Properties properties) {
+        KafkaProducerConfig kafkaProducerConfig = (KafkaProducerConfig) this.mqProperties;
+        Map<String, Object> kafkaProperties = kafkaProducerConfig.getKafkaProperties();
+        for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+            String key = (String) entry.getKey();
+            Object value = entry.getValue();
+            if (key.startsWith(PREFIX_KAFKA_CONFIG) && value != null) {
+                key = key.substring(PREFIX_KAFKA_CONFIG.length());
+                kafkaProperties.put(key, value);
+            }
+        }
+
+        String kerberosEnabled = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
+        if (!StringUtils.isEmpty(kerberosEnabled)) {
+            kafkaProducerConfig.setKerberosEnabled(Boolean.parseBoolean(kerberosEnabled));
+        }
+        String krb5File = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE);
+        if (!StringUtils.isEmpty(krb5File)) {
+            kafkaProducerConfig.setKrb5File(krb5File);
+        }
+        String jaasFile = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE);
+        if (!StringUtils.isEmpty(jaasFile)) {
+            kafkaProducerConfig.setJaasFile(jaasFile);
+        }
+    }
+
+    @Override
+    public void stop() {
+        try {
+            logger.info("## stop the kafka producer");
+            if (producer != null) {
+                producer.close();
+            }
+            super.stop();
+        } catch (Throwable e) {
+            logger.warn("##something goes wrong when stopping kafka producer:", e);
+        } finally {
+            logger.info("## kafka producer is down.");
+        }
+    }
+
+    @Override
+    public void send(MQDestination mqDestination, Message message, Callback callback) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+
+        try {
+            List result;
+            if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) {
+                // 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
+                Map<String, Message> messageMap = MQMessageUtils
+                    .messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic());
+
+                // 针对不同的topic,引入多线程提升效率
+                for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
+                    final String topicName = entry.getKey().replace('.', '_');
+                    final Message messageSub = entry.getValue();
+                    template.submit((Callable) () -> {
+                        try {
+                            return send(mqDestination, topicName, messageSub, mqProperties.isFlatMessage());
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+
+                result = template.waitForResult();
+            } else {
+                result = new ArrayList();
+                List<Future> futures = send(mqDestination,
+                    mqDestination.getTopic(),
+                    message,
+                    mqProperties.isFlatMessage());
+                result.add(futures);
+            }
+
+            // 一个批次的所有topic和分区的队列，都采用异步的模式进行多线程批量发送
+            // 最后在集结点进行flush等待，确保所有数据都写出成功
+            // 注意：kafka的异步模式如果要保证顺序性，需要设置max.in.flight.requests.per.connection=1，确保在网络异常重试时有排他性
+            producer.flush();
+            // flush操作也有可能是发送失败,这里需要异步关注一下发送结果,针对有异常的直接出发rollback
+            for (Object obj : result) {
+                List<Future> futures = (List<Future>) obj;
+                for (Future future : futures) {
+                    try {
+                        future.get();
+                    } catch (InterruptedException | ExecutionException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
+        List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
+        if (!flat) {
+            if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
+                // 并发构造
+                EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
+                // 串行分区
+                Message[] messages = MQMessageUtils.messagePartition(datas,
+                    message.getId(),
+                    mqDestination.getPartitionsNum(),
+                    mqDestination.getPartitionHash(),
+                    this.mqProperties.isDatabaseHash());
+                int length = messages.length;
+                for (int i = 0; i < length; i++) {
+                    Message messagePartition = messages[i];
+                    if (messagePartition != null) {
+                        records.add(new ProducerRecord<>(topicName,
+                            i,
+                            null,
+                            CanalMessageSerializerUtil.serializer(messagePartition, true)));
+                    }
+                }
+            } else {
+                final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
+                records.add(new ProducerRecord<>(topicName,
+                    partition,
+                    null,
+                    CanalMessageSerializerUtil.serializer(message, true)));
+            }
+        } else {
+            // 发送扁平数据json
+            // 并发构造
+            EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
+            for (FlatMessage flatMessage : flatMessages) {
+                if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
+                    FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
+                        mqDestination.getPartitionsNum(),
+                        mqDestination.getPartitionHash(),
+                        this.mqProperties.isDatabaseHash());
+                    int length = partitionFlatMessage.length;
+                    for (int i = 0; i < length; i++) {
+                        FlatMessage flatMessagePart = partitionFlatMessage[i];
+                        if (flatMessagePart != null) {
+                            records.add(new ProducerRecord<>(topicName,
+                                i,
+                                null,
+                                JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
+                        }
+                    }
+                } else {
+                    final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
+                    records.add(new ProducerRecord<>(topicName,
+                        partition,
+                        null,
+                        JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
+                }
+            }
+        }
+
+        return produce(records);
+    }
+
+    private List<Future> produce(List<ProducerRecord<String, byte[]>> records) {
+        List<Future> futures = new ArrayList<>();
+        // 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题
+        for (ProducerRecord record : records) {
+            futures.add(producer.send(record));
+        }
+
+        return futures;
+    }
+
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java

@@ -0,0 +1,28 @@
+package com.alibaba.otter.canal.connector.kafka.producer;
+
+import org.apache.kafka.common.serialization.Serializer;
+
+import java.util.Map;
+
+/**
+ * Kafka Message类的序列化
+ *
+ * @author rewerma 2018-6-11 下午05:30:49
+ * @version 1.0.0
+ */
+public class KafkaMessageSerializer implements Serializer<byte[]> {
+
+    @Override
+    public void configure(Map<String, ?> configs, boolean isKey) {
+    }
+
+    @Override
+    public byte[] serialize(String topic, byte[] data) {
+        return data;
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+kafka=com.alibaba.otter.canal.connector.kafka.producer.CanalKafkaProducer

connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+kafka=com.alibaba.otter.canal.connector.kafka.consumer.CanalKafkaConsumer

connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.connector.kafka.test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
+
+@Ignore
+public class CanalKafkaProducerTest {
+
+    @Test
+    public void testLoadKafkaProducer() throws IOException {
+        Properties pro = new Properties();
+        FileInputStream in = new FileInputStream("../../deployer/src/main/resources/canal.properties");
+        pro.load(in);
+
+        ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
+        CanalMQProducer canalMQProducer = loader
+            .getExtension("kafka", "/../../deployer/target/canal/plugin", "/../../deployer/target/canal/plugin");
+        canalMQProducer.init(pro);
+
+        in.close();
+    }
+}

connector/pom.xml

@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.alibaba.otter</groupId>
+        <artifactId>canal</artifactId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>canal.connector</artifactId>
+    <version>1.1.5-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <name>canal connector module for otter ${project.version}</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.test.skip>true</maven.test.skip>
+        <downloadSources>true</downloadSources>
+        <java_source_version>1.8</java_source_version>
+        <java_target_version>1.8</java_target_version>
+        <file_encoding>UTF-8</file_encoding>
+    </properties>
+
+    <licenses>
+        <license>
+            <name>Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+        </license>
+    </licenses>
+
+    <scm>
+        <url>git@github.com:alibaba/canal.git</url>
+        <connection>scm:git:git@github.com:alibaba/canal.git</connection>
+        <developerConnection>scm:git:git@github.com:alibaba/canal.git</developerConnection>
+    </scm>
+
+    <repositories>
+        <repository>
+            <id>central</id>
+            <url>http://repo1.maven.org/maven2</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>java.net</id>
+            <url>http://download.java.net/maven/2/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>aliyun</id>
+            <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>sonatype</id>
+            <name>sonatype</name>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>sonatype-release</id>
+            <name>sonatype-release</name>
+            <url>https://oss.sonatype.org/service/local/repositories/releases/content</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <modules>
+        <module>kafka-connector</module>
+        <module>core</module>
+        <module>rocketmq-connector</module>
+        <module>rabbitmq-connector</module>
+        <module>tcp-connector</module>
+    </modules>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.7.0</version>
+                <configuration>
+                    <source>${java_source_version}</source>
+                    <target>${java_target_version}</target>
+                    <encoding>${file_encoding}</encoding>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>sonatype-nexus-snapshots</id>
+            <name>Sonatype Nexus Snapshots</name>
+            <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
+        </snapshotRepository>
+        <repository>
+            <id>sonatype-nexus-staging</id>
+            <name>Nexus Release Repository</name>
+            <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
+    </distributionManagement>
+</project>

connector/rabbitmq-connector/pom.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.rabbitmq</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector rabbitmq module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.rabbitmq</groupId>
+            <artifactId>amqp-client</artifactId>
+            <version>5.5.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.mq-amqp</groupId>
+            <artifactId>mq-amqp-client</artifactId>
+            <version>1.0.3</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java

@@ -0,0 +1,20 @@
+package com.alibaba.otter.canal.connector.rabbitmq.config;
+
+/**
+ * RabbitMQ 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RabbitMQConstants {
+
+    public static final String ROOT                      = "rabbitmq";
+
+    public static final String RABBITMQ_HOST             = ROOT + "." + "host";
+    public static final String RABBITMQ_EXCHANGE         = ROOT + "." + "exchange";
+    public static final String RABBITMQ_VIRTUAL_HOST     = ROOT + "." + "virtual.host";
+    public static final String RABBITMQ_USERNAME         = ROOT + "." + "username";
+    public static final String RABBITMQ_PASSWORD         = ROOT + "." + "password";
+
+    public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId";
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java

@@ -0,0 +1,58 @@
+package com.alibaba.otter.canal.connector.rabbitmq.config;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * RabbitMQ 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RabbitMQProducerConfig extends MQProperties {
+
+    private String host;
+    private String virtualHost;
+    private String exchange;
+    private String username;
+    private String password;
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public String getVirtualHost() {
+        return virtualHost;
+    }
+
+    public void setVirtualHost(String virtualHost) {
+        this.virtualHost = virtualHost;
+    }
+
+    public String getExchange() {
+        return exchange;
+    }
+
+    public void setExchange(String exchange) {
+        this.exchange = exchange;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java

@@ -0,0 +1,214 @@
+package com.alibaba.otter.canal.connector.rabbitmq.consumer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
+import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+import com.rabbitmq.client.*;
+
+/**
+ * RabbitMQ consumer SPI 实现
+ *
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("rabbitmq")
+public class CanalRabbitMQConsumer implements CanalMsgConsumer {
+
+    private static final Logger                                logger              = LoggerFactory
+        .getLogger(CanalRabbitMQConsumer.class);
+
+    // 链接地址
+    private String                                             nameServer;
+    // 主机名
+    private String                                             vhost;
+    private String                                             queueName;
+
+    // 一些鉴权信息
+    private String                                             accessKey;
+    private String                                             secretKey;
+    private Long                                               resourceOwnerId;
+    private String                                             username;
+    private String                                             password;
+
+    private boolean                                            flatMessage;
+
+    private Connection                                         connect;
+    private Channel                                            channel;
+
+    private long                                               batchProcessTimeout = 60 * 1000;
+    private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
+    private volatile ConsumerBatchMessage<CommonMessage>       lastGetBatchMessage = null;
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        this.nameServer = properties.getProperty("rabbitmq.host");
+        this.vhost = properties.getProperty("rabbitmq.virtual.host");
+        this.queueName = topic;
+        this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        this.username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
+        this.password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
+        Long resourceOwnerIdPro = (Long) properties.get(RabbitMQConstants.RABBITMQ_RESOURCE_OWNERID);
+        if (resourceOwnerIdPro != null) {
+            this.resourceOwnerId = resourceOwnerIdPro;
+        }
+        this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
+    }
+
+    @Override
+    public void connect() {
+        ConnectionFactory factory = new ConnectionFactory();
+        if (accessKey.length() > 0 && secretKey.length() > 0) {
+            factory.setCredentialsProvider(new AliyunCredentialsProvider(accessKey, secretKey, resourceOwnerId));
+        } else {
+            factory.setUsername(username);
+            factory.setPassword(password);
+        }
+        factory.setHost(nameServer);
+        factory.setAutomaticRecoveryEnabled(true);
+        factory.setNetworkRecoveryInterval(5000);
+        factory.setVirtualHost(vhost);
+        try {
+            connect = factory.newConnection();
+            channel = connect.createChannel();
+        } catch (IOException | TimeoutException e) {
+            throw new CanalClientException("Start RabbitMQ producer error", e);
+        }
+
+        // 不存在连接 则重新连接
+        if (connect == null) {
+            this.connect();
+        }
+
+        Consumer consumer = new DefaultConsumer(channel) {
+
+            @Override
+            public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties,
+                                       byte[] body) throws IOException {
+
+                if (body != null) {
+                    channel.basicAck(envelope.getDeliveryTag(), process(body));
+                }
+            }
+        };
+        try {
+            channel.basicConsume(queueName, false, consumer);
+        } catch (IOException e) {
+            throw new CanalClientException("error", e);
+        }
+    }
+
+    private boolean process(byte[] messageData) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Get Message: {}", new String(messageData));
+        }
+        List<CommonMessage> messageList = Lists.newArrayList();
+        if (!flatMessage) {
+            Message message = CanalMessageSerializerUtil.deserializer(messageData);
+            messageList.addAll(MessageUtil.convert(message));
+        } else {
+            CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
+            messageList.add(commonMessage);
+        }
+        ConsumerBatchMessage<CommonMessage>  batchMessage = new ConsumerBatchMessage<>(messageList);
+        try {
+            messageBlockingQueue.put(batchMessage);
+        } catch (InterruptedException e) {
+            logger.error("Put message to queue error", e);
+            throw new RuntimeException(e);
+        }
+        boolean isCompleted;
+        try {
+            isCompleted = batchMessage.waitFinish(batchProcessTimeout);
+        } catch (InterruptedException e) {
+            logger.error("Interrupted when waiting messages to be finished.", e);
+            throw new RuntimeException(e);
+        }
+        boolean isSuccess = batchMessage.isSuccess();
+        return isCompleted && isSuccess;
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
+            }
+
+            ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
+            if (batchMessage != null) {
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
+            }
+        } catch (InterruptedException ex) {
+            logger.warn("Get message timeout", ex);
+            throw new CanalClientException("Failed to fetch the data after: " + timeout);
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.ack();
+            }
+        } catch (Throwable e) {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        if (connect != null) {
+            try {
+                connect.close();
+            } catch (IOException e) {
+                throw new CanalClientException("stop connect error", e);
+            }
+        }
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException | TimeoutException e) {
+                throw new CanalClientException("stop channel error", e);
+            }
+        }
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.connector.rabbitmq.consumer;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class ConsumerBatchMessage<T> {
+
+    private final List<T>  data;
+    private CountDownLatch         latch;
+    private boolean                hasFailure = false;
+
+    public ConsumerBatchMessage(List<T> data){
+        this.data = data;
+        latch = new CountDownLatch(1);
+    }
+
+    public boolean waitFinish(long timeout) throws InterruptedException {
+        return latch.await(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    public boolean isSuccess() {
+        return !hasFailure;
+    }
+
+    public List<T> getData() {
+        return data;
+    }
+
+    /**
+     * Countdown if the sub message is successful.
+     */
+    public void ack() {
+        latch.countDown();
+    }
+
+    /**
+     * Countdown and fail-fast if the sub message is failed.
+     */
+    public void fail() {
+        hasFailure = true;
+        // fail fast
+        long count = latch.getCount();
+        for (int i = 0; i < count; i++) {
+            latch.countDown();
+        }
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.connector.rabbitmq.producer;
+
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
+import com.alibaba.mq.amqp.utils.UserUtils;
+import com.rabbitmq.client.impl.CredentialsProvider;
+
+/**
+ * aliyun amqp协议 账号类
+ * 暂不支持STS授权情况
+ *
+ * @version 1.0.0
+ */
+public class AliyunCredentialsProvider implements CredentialsProvider {
+
+    /**
+     * Access Key ID
+     */
+    private final String AliyunAccessKey;
+
+    /**
+     * Access Key Secret
+     */
+    private final String AliyunAccessSecret;
+
+    /**
+     * 资源主账号ID
+     */
+    private final long   resourceOwnerId;
+
+    public AliyunCredentialsProvider(final String accessKey, final String accessSecret, final long resourceOwnerId){
+        this.AliyunAccessKey = accessKey;
+        this.AliyunAccessSecret = accessSecret;
+        this.resourceOwnerId = resourceOwnerId;
+    }
+
+    @Override
+    public String getUsername() {
+        return UserUtils.getUserName(AliyunAccessKey, resourceOwnerId);
+    }
+
+    @Override
+    public String getPassword() {
+        try {
+            return UserUtils.getPassord(AliyunAccessSecret);
+        } catch (InvalidKeyException | NoSuchAlgorithmException ignored) {
+        }
+        return null;
+    }
+
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java

@@ -0,0 +1,180 @@
+package com.alibaba.otter.canal.connector.rabbitmq.producer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+/**
+ * RabbitMQ Producer SPI 实现
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("rabbitmq")
+public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
+
+    private Connection          connect;
+    private Channel             channel;
+
+    @Override
+    public void init(Properties properties) {
+        RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig();
+        this.mqProperties = rabbitMQProperties;
+        super.init(properties);
+        loadRabbitMQProperties(properties);
+
+        ConnectionFactory factory = new ConnectionFactory();
+        factory.setHost(rabbitMQProperties.getHost());
+        if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
+            && mqProperties.getAliyunUid() > 0) {
+            factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
+                mqProperties.getAliyunSecretKey(),
+                mqProperties.getAliyunUid()));
+        } else {
+            factory.setUsername(rabbitMQProperties.getUsername());
+            factory.setPassword(rabbitMQProperties.getPassword());
+        }
+        factory.setVirtualHost(rabbitMQProperties.getVirtualHost());
+        try {
+            connect = factory.newConnection();
+            channel = connect.createChannel();
+            // channel.exchangeDeclare(mqProperties.getExchange(), "topic");
+        } catch (IOException | TimeoutException ex) {
+            throw new CanalException("Start RabbitMQ producer error", ex);
+        }
+    }
+
+    private void loadRabbitMQProperties(Properties properties) {
+        RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
+
+        String host = properties.getProperty(RabbitMQConstants.RABBITMQ_HOST);
+        if (!StringUtils.isEmpty(host)) {
+            rabbitMQProperties.setHost(host);
+        }
+        String vhost = properties.getProperty(RabbitMQConstants.RABBITMQ_VIRTUAL_HOST);
+        if (!StringUtils.isEmpty(vhost)) {
+            rabbitMQProperties.setVirtualHost(vhost);
+        }
+        String exchange = properties.getProperty(RabbitMQConstants.RABBITMQ_EXCHANGE);
+        if (!StringUtils.isEmpty(exchange)) {
+            rabbitMQProperties.setExchange(exchange);
+        }
+        String username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
+        if (!StringUtils.isEmpty(username)) {
+            rabbitMQProperties.setUsername(username);
+        }
+        String password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
+        if (!StringUtils.isEmpty(password)) {
+            rabbitMQProperties.setPassword(password);
+        }
+    }
+
+    @Override
+    public void send(final MQDestination destination, Message message, Callback callback) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map<String, Message> messageMap = MQMessageUtils
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
+
+                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                    final String topicName = entry.getKey().replace('.', '_');
+                    final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+
+                    template.submit(new Runnable() {
+
+                        @Override
+                        public void run() {
+                            send(destination, topicName, messageSub);
+                        }
+                    });
+                }
+
+                template.waitForResult();
+            } else {
+                send(destination, destination.getTopic(), message);
+            }
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    private void send(MQDestination canalDestination, String topicName, Message messageSub) {
+        if (!mqProperties.isFlatMessage()) {
+            byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry());
+            if (logger.isDebugEnabled()) {
+                logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
+            }
+            sendMessage(topicName, message);
+        } else {
+            // 并发构造
+            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
+            for (FlatMessage flatMessage : flatMessages) {
+                byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
+                if (logger.isDebugEnabled()) {
+                    logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
+                }
+                sendMessage(topicName, message);
+            }
+        }
+
+    }
+
+    private void sendMessage(String queueName, byte[] message) {
+        // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
+        try {
+            RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
+            channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message);
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RabbitMQ producer##");
+        try {
+            this.connect.close();
+            this.channel.close();
+            super.stop();
+        } catch (IOException | TimeoutException ex) {
+            throw new CanalException("Stop RabbitMQ producer error", ex);
+        }
+
+        super.stop();
+    }
+}

connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.producer.CanalRabbitMQProducer

connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.consumer.CanalRabbitMQConsumer

connector/rocketmq-connector/pom.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.rocketmq</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector rocketMQ module for otter ${project.version}</name>
+
+    <properties>
+        <rocketmq_version>4.5.2</rocketmq_version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-client</artifactId>
+            <version>${rocketmq_version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-acl</artifactId>
+            <version>${rocketmq_version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar"/>
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java

@@ -0,0 +1,25 @@
+package com.alibaba.otter.canal.connector.rocketmq.config;
+
+/**
+ * RocketMQ 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RocketMQConstants {
+
+    public static final String ROOT                                  = "rocketmq";
+
+    public static final String ROCKETMQ_PRODUCER_GROUP               = ROOT + "." + "producer.group";
+    public static final String ROCKETMQ_ENABLE_MESSAGE_TRACE         = ROOT + "." + "enable.message.trace";
+    public static final String ROCKETMQ_CUSTOMIZED_TRACE_TOPIC       = ROOT + "." + "customized.trace.topic";
+    public static final String ROCKETMQ_NAMESPACE                    = ROOT + "." + "namespace";
+    public static final String ROCKETMQ_NAMESRV_ADDR                 = ROOT + "." + "namesrv.addr";
+    public static final String ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED = ROOT + "." + "retry.times.when.send.failed";
+    public static final String ROCKETMQ_VIP_CHANNEL_ENABLED          = ROOT + "." + "vip.channel.enabled";
+
+    public static final String ROCKETMQ_ACCESS_CHANNEL               = ROOT + "." + "access.channel";
+    public static final String ROCKETMQ_BATCH_SIZE                   = ROOT + "." + "batch.size";
+    public static final String ROCKETMQ_SUBSCRIBE_FILTER             = ROOT + "." + "subscribe.filter";
+
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java

@@ -0,0 +1,76 @@
+package com.alibaba.otter.canal.connector.rocketmq.config;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * RocketMQ 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RocketMQProducerConfig extends MQProperties {
+
+    private String  producerGroup;
+    private boolean enableMessageTrace       = false;
+    private String  customizedTraceTopic;
+    private String  namespace;
+    private String  namesrvAddr;
+    private int     retryTimesWhenSendFailed = 0;
+    private boolean vipChannelEnabled        = false;
+
+    public String getProducerGroup() {
+        return producerGroup;
+    }
+
+    public void setProducerGroup(String producerGroup) {
+        this.producerGroup = producerGroup;
+    }
+
+    public boolean isEnableMessageTrace() {
+        return enableMessageTrace;
+    }
+
+    public void setEnableMessageTrace(boolean enableMessageTrace) {
+        this.enableMessageTrace = enableMessageTrace;
+    }
+
+    public String getCustomizedTraceTopic() {
+        return customizedTraceTopic;
+    }
+
+    public void setCustomizedTraceTopic(String customizedTraceTopic) {
+        this.customizedTraceTopic = customizedTraceTopic;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    public String getNamesrvAddr() {
+        return namesrvAddr;
+    }
+
+    public void setNamesrvAddr(String namesrvAddr) {
+        this.namesrvAddr = namesrvAddr;
+    }
+
+    public int getRetryTimesWhenSendFailed() {
+        return retryTimesWhenSendFailed;
+    }
+
+    public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) {
+        this.retryTimesWhenSendFailed = retryTimesWhenSendFailed;
+    }
+
+    public boolean isVipChannelEnabled() {
+        return vipChannelEnabled;
+    }
+
+    public void setVipChannelEnabled(boolean vipChannelEnabled) {
+        this.vipChannelEnabled = vipChannelEnabled;
+    }
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java

@@ -0,0 +1,237 @@
+package com.alibaba.otter.canal.connector.rocketmq.consumer;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.AccessChannel;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+
+/**
+ * RocketMQ consumer SPI 实现
+ *
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("rocketmq")
+public class CanalRocketMQConsumer implements CanalMsgConsumer {
+
+    private static final Logger                                logger               = LoggerFactory
+        .getLogger(CanalRocketMQConsumer.class);
+    private static final String                                CLOUD_ACCESS_CHANNEL = "cloud";
+
+    private String                                             nameServer;
+    private String                                             topic;
+    private String                                             groupName;
+    private volatile boolean                                   connected            = false;
+    private DefaultMQPushConsumer                              rocketMQConsumer;
+    private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
+    private int                                                batchSize            = -1;
+    private long                                               batchProcessTimeout  = 60 * 1000;
+    private boolean                                            flatMessage;
+    private volatile ConsumerBatchMessage<CommonMessage>       lastGetBatchMessage  = null;
+    private String                                             accessKey;
+    private String                                             secretKey;
+    private String                                             customizedTraceTopic;
+    private boolean                                            enableMessageTrace   = false;
+    private String                                             accessChannel;
+    private String                                             namespace;
+    private String                                             filter               = "*";
+
+    @Override
+    public void init(Properties properties, String topic, String groupName) {
+        this.topic = topic;
+        this.groupName = groupName;
+        this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
+        this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
+        if (StringUtils.isNotEmpty(enableMessageTrace)) {
+            this.enableMessageTrace = Boolean.parseBoolean(enableMessageTrace);
+        }
+        this.customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
+        this.accessChannel = properties.getProperty(RocketMQConstants.ROCKETMQ_ACCESS_CHANNEL);
+        this.namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
+        this.nameServer = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
+        String batchSize = properties.getProperty(RocketMQConstants.ROCKETMQ_BATCH_SIZE);
+        if (StringUtils.isNotEmpty(batchSize)) {
+            this.batchSize = Integer.parseInt(batchSize);
+        }
+        String subscribeFilter = properties.getProperty(RocketMQConstants.ROCKETMQ_SUBSCRIBE_FILTER);
+        if (StringUtils.isNotEmpty(subscribeFilter)) {
+            this.filter = subscribeFilter;
+        }
+    }
+
+    @Override
+    public void connect() {
+        RPCHook rpcHook = null;
+        if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
+            SessionCredentials sessionCredentials = new SessionCredentials();
+            sessionCredentials.setAccessKey(accessKey);
+            sessionCredentials.setSecretKey(secretKey);
+            rpcHook = new AclClientRPCHook(sessionCredentials);
+        }
+
+        rocketMQConsumer = new DefaultMQPushConsumer(groupName,
+            rpcHook,
+            new AllocateMessageQueueAveragely(),
+            enableMessageTrace,
+            customizedTraceTopic);
+        rocketMQConsumer.setVipChannelEnabled(false);
+        if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) {
+            rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD);
+        }
+
+        if (!StringUtils.isEmpty(this.namespace)) {
+            rocketMQConsumer.setNamespace(this.namespace);
+        }
+
+        if (!StringUtils.isBlank(nameServer)) {
+            rocketMQConsumer.setNamesrvAddr(nameServer);
+        }
+        if (batchSize != -1) {
+            rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
+        }
+
+        try {
+            if (rocketMQConsumer == null) {
+                this.connect();
+            }
+            rocketMQConsumer.subscribe(this.topic, this.filter);
+            rocketMQConsumer.registerMessageListener((MessageListenerOrderly) (messageExts, context) -> {
+                context.setAutoCommit(true);
+                boolean isSuccess = process(messageExts);
+                if (isSuccess) {
+                    return ConsumeOrderlyStatus.SUCCESS;
+                } else {
+                    return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
+                }
+            });
+            rocketMQConsumer.start();
+        } catch (MQClientException ex) {
+            connected = false;
+            logger.error("Start RocketMQ consumer error", ex);
+        }
+    }
+
+    private boolean process(List<MessageExt> messageExts) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Get Message: {}", messageExts);
+        }
+        List<CommonMessage> messageList = Lists.newArrayList();
+        for (MessageExt messageExt : messageExts) {
+            byte[] data = messageExt.getBody();
+            if (data != null) {
+                try {
+                    if (!flatMessage) {
+                        Message message = CanalMessageSerializerUtil.deserializer(data);
+                        messageList.addAll(MessageUtil.convert(message));
+                    } else {
+                        CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
+                        messageList.add(commonMessage);
+                    }
+                } catch (Exception ex) {
+                    logger.error("Add message error", ex);
+                    throw new CanalClientException(ex);
+                }
+            } else {
+                logger.warn("Received message data is null");
+            }
+        }
+        ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
+        try {
+            messageBlockingQueue.put(batchMessage);
+        } catch (InterruptedException e) {
+            logger.error("Put message to queue error", e);
+            throw new RuntimeException(e);
+        }
+        boolean isCompleted;
+        try {
+            isCompleted = batchMessage.waitFinish(batchProcessTimeout);
+        } catch (InterruptedException e) {
+            logger.error("Interrupted when waiting messages to be finished.", e);
+            throw new RuntimeException(e);
+        }
+        boolean isSuccess = batchMessage.isSuccess();
+        return isCompleted && isSuccess;
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
+            }
+
+            ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
+            if (batchMessage != null) {
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
+            }
+        } catch (InterruptedException ex) {
+            logger.warn("Get message timeout", ex);
+            throw new CanalClientException("Failed to fetch the data after: " + timeout);
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.ack();
+            }
+        } catch (Throwable e) {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        rocketMQConsumer.unsubscribe(topic);
+        rocketMQConsumer.shutdown();
+        connected = false;
+    }
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.connector.rocketmq.consumer;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class ConsumerBatchMessage<T> {
+
+    private final List<T>  data;
+    private CountDownLatch         latch;
+    private boolean                hasFailure = false;
+
+    public ConsumerBatchMessage(List<T> data){
+        this.data = data;
+        latch = new CountDownLatch(1);
+    }
+
+    public boolean waitFinish(long timeout) throws InterruptedException {
+        return latch.await(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    public boolean isSuccess() {
+        return !hasFailure;
+    }
+
+    public List<T> getData() {
+        return data;
+    }
+
+    /**
+     * Countdown if the sub message is successful.
+     */
+    public void ack() {
+        latch.countDown();
+    }
+
+    /**
+     * Countdown and fail-fast if the sub message is failed.
+     */
+    public void fail() {
+        hasFailure = true;
+        // fail fast
+        long count = latch.getCount();
+        for (int i = 0; i < count; i++) {
+            latch.countDown();
+        }
+    }
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java

@@ -0,0 +1,312 @@
+package com.alibaba.otter.canal.connector.rocketmq.producer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.AccessChannel;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
+import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+
+/**
+ * RocketMQ Producer SPI 实现
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("rocketmq")
+public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger logger               = LoggerFactory.getLogger(CanalRocketMQProducer.class);
+
+    private DefaultMQProducer   defaultMQProducer;
+    private static final String CLOUD_ACCESS_CHANNEL = "cloud";
+
+    @Override
+    public void init(Properties properties) {
+        RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
+        this.mqProperties = rocketMQProperties;
+        super.init(properties);
+        loadRocketMQProperties(properties);
+
+        RPCHook rpcHook = null;
+        if (mqProperties.getAliyunAccessKey().length() > 0
+            && mqProperties.getAliyunSecretKey().length() > 0) {
+            SessionCredentials sessionCredentials = new SessionCredentials();
+            sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
+            sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
+            rpcHook = new AclClientRPCHook(sessionCredentials);
+        }
+
+        defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
+            rpcHook,
+            rocketMQProperties.isEnableMessageTrace(),
+            rocketMQProperties.getCustomizedTraceTopic());
+        if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
+            defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
+        }
+        if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) {
+            defaultMQProducer.setNamespace(rocketMQProperties.getNamespace());
+        }
+        defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr());
+        defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed());
+        defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled());
+        logger.info("##Start RocketMQ producer##");
+        try {
+            defaultMQProducer.start();
+        } catch (MQClientException ex) {
+            throw new CanalException("Start RocketMQ producer error", ex);
+        }
+    }
+
+    private void loadRocketMQProperties(Properties properties) {
+        RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
+
+        String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
+        if (!StringUtils.isEmpty(producerGroup)) {
+            rocketMQProperties.setProducerGroup(producerGroup);
+        }
+        String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
+        if (!StringUtils.isEmpty(enableMessageTrace)) {
+            rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
+        }
+        String customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
+        if (!StringUtils.isEmpty(customizedTraceTopic)) {
+            rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
+        }
+        String namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
+        if (!StringUtils.isEmpty(namespace)) {
+            rocketMQProperties.setNamespace(namespace);
+        }
+        String namesrvAddr = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
+        if (!StringUtils.isEmpty(namesrvAddr)) {
+            rocketMQProperties.setNamesrvAddr(namesrvAddr);
+        }
+        String retry = properties.getProperty(RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED);
+        if (!StringUtils.isEmpty(retry)) {
+            rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
+        }
+        String vipChannelEnabled = properties.getProperty(RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
+        if (!StringUtils.isEmpty(vipChannelEnabled)) {
+            rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
+        }
+    }
+
+    @Override
+    public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
+
+                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                    String topicName = entry.getKey().replace('.', '_');
+                    com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+                    template.submit(() -> {
+                        try {
+                            send(destination, topicName, messageSub);
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+
+                template.waitForResult();
+            } else {
+                send(destination, destination.getTopic(), message);
+            }
+
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    public void send(final MQDestination destination, String topicName,
+                     com.alibaba.otter.canal.protocol.Message message) {
+        if (!mqProperties.isFlatMessage()) {
+            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                // 并发构造
+                MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
+                // 串行分区
+                com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
+                    message.getId(),
+                    destination.getPartitionsNum(),
+                    destination.getPartitionHash(),
+                    mqProperties.isDatabaseHash());
+                int length = messages.length;
+
+                ExecutorTemplate template = new ExecutorTemplate(executor);
+                for (int i = 0; i < length; i++) {
+                    com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
+                    if (dataPartition != null) {
+                        final int index = i;
+                        template.submit(() -> {
+                            Message data = new Message(topicName,
+                                CanalMessageSerializerUtil.serializer(dataPartition,
+                                    mqProperties.isFilterTransactionEntry()));
+                            sendMessage(data, index);
+                        });
+                    }
+                }
+                // 等所有分片发送完毕
+                template.waitForResult();
+            } else {
+                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
+                Message data = new Message(topicName,
+                    CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
+                sendMessage(data, partition);
+            }
+        } else {
+            // 并发构造
+            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
+            // 初始化分区合并队列
+            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
+                for (int i = 0; i < destination.getPartitionsNum(); i++) {
+                    partitionFlatMessages.add(new ArrayList<>());
+                }
+
+                for (FlatMessage flatMessage : flatMessages) {
+                    FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
+                        destination.getPartitionsNum(),
+                        destination.getPartitionHash(),
+                        mqProperties.isDatabaseHash());
+                    int length = partitionFlatMessage.length;
+                    for (int i = 0; i < length; i++) {
+                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
+                    }
+                }
+
+                ExecutorTemplate template = new ExecutorTemplate(executor);
+                for (int i = 0; i < partitionFlatMessages.size(); i++) {
+                    final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
+                    if (flatMessagePart != null) {
+                        final int index = i;
+                        template.submit(() -> {
+                            List<Message> messages = flatMessagePart.stream()
+                                .map(flatMessage -> new Message(topicName,
+                                    JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
+                                .collect(Collectors.toList());
+                            // 批量发送
+                            sendMessage(messages, index);
+                        });
+                    }
+                }
+
+                // 批量等所有分区的结果
+                template.waitForResult();
+            } else {
+                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
+                List<Message> messages = flatMessages.stream()
+                    .map(flatMessage -> new Message(topicName,
+                        JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
+                    .collect(Collectors.toList());
+                // 批量发送
+                sendMessage(messages, partition);
+            }
+        }
+    }
+
+    private void sendMessage(Message message, int partition) {
+        try {
+            SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> {
+                if (partition > mqs.size()) {
+                    return mqs.get(partition % mqs.size());
+                } else {
+                    return mqs.get(partition);
+                }
+            }, null);
+
+            if (logger.isDebugEnabled()) {
+                logger.debug("Send Message Result: {}", sendResult);
+            }
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @SuppressWarnings("deprecation")
+    private void sendMessage(List<Message> messages, int partition) {
+        if (messages.isEmpty()) {
+            return;
+        }
+
+        // 获取一下messageQueue
+        DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
+        TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
+        if (topicInfo == null) {
+            for (Message message : messages) {
+                sendMessage(message, partition);
+            }
+        } else {
+            // 批量发送
+            List<MessageQueue> queues = topicInfo.getMessageQueueList();
+            int size = queues.size();
+            if (size <= 0) {
+                // 可能是第一次创建
+                for (Message message : messages) {
+                    sendMessage(message, partition);
+                }
+            } else {
+                MessageQueue queue;
+                if (partition > size) {
+                    queue = queues.get(partition % size);
+                } else {
+                    queue = queues.get(partition);
+                }
+
+                try {
+                    SendResult sendResult = this.defaultMQProducer.send(messages, queue);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Send Message Result: {}", sendResult);
+                    }
+                } catch (Throwable e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RocketMQ producer##");
+        this.defaultMQProducer.shutdown();
+        super.stop();
+    }
+}

connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+rocketmq=com.alibaba.otter.canal.connector.rocketmq.producer.CanalRocketMQProducer

connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+rocketmq=com.alibaba.otter.canal.connector.rocketmq.consumer.CanalRocketMQConsumer

connector/tcp-connector/pom.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.tcp</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector tcp module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.client</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-orm</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jdbc</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-aop</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-context</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.rocketmq</groupId>
+                    <artifactId>rocketmq-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.aliyun.openservices</groupId>
+                    <artifactId>aliware-apache-rocketmq-cloud</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin"
+                                      overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.tcp-${project.version}-jar-with-dependencies.jar"/>
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java

@@ -0,0 +1,12 @@
+package com.alibaba.otter.canal.connector.tcp.config;
+
+public class TCPConstants {
+
+    public static final String ROOT                 = "canal";
+
+    public static final String CANAL_TCP_HOST       = ROOT + "." + "tcp.server.host";
+    public static final String CANAL_TCP_ZK_HOSTS   = ROOT + "." + "tcp.zookeeper.hosts";
+    public static final String CANAL_TCP_USERNAME   = ROOT + "." + "tcp.username";
+    public static final String CANAL_TCP_PASSWORD   = ROOT + "." + "tcp.password";
+    public static final String CANAL_TCP_BATCH_SIZE = ROOT + "." + "tcp.batch.size";
+}

connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java

@@ -0,0 +1,106 @@
+package com.alibaba.otter.canal.connector.tcp.consumer;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
+import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
+import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
+import com.alibaba.otter.canal.protocol.Message;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * TCP 消费者连接器, 一个destination对应一个SPI实例
+ * 
+ * @author rewerma 2020-01-30
+ * @version 1.0.0
+ */
+@SPI("tcp")
+public class CanalTCPConsumer implements CanalMsgConsumer {
+
+    private static final String PREFIX_TCP_CONFIG = "canal.tcp.";
+
+    private Long                currentBatchId    = null;
+
+    private CanalConnector      canalConnector;
+
+    private int                 batchSize         = 500;
+
+    @Override
+    public void init(Properties properties, String destination, String groupId) {
+        // load config
+        String host = properties.getProperty(TCPConstants.CANAL_TCP_HOST);
+        String username = properties.getProperty(TCPConstants.CANAL_TCP_USERNAME);
+        String password = properties.getProperty(TCPConstants.CANAL_TCP_PASSWORD);
+        String zkHosts = properties.getProperty(TCPConstants.CANAL_TCP_ZK_HOSTS);
+        String batchSizePro = properties.getProperty(TCPConstants.CANAL_TCP_BATCH_SIZE);
+        if (batchSizePro != null) {
+            batchSize = Integer.parseInt(batchSizePro);
+        }
+        if (host != null) {
+            String[] ipPort = host.split(":");
+            SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
+            this.canalConnector = new SimpleCanalConnector(sa, username, password, destination);
+        } else {
+            this.canalConnector = new ClusterCanalConnector(username,
+                password,
+                destination,
+                new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkHosts)));
+        }
+    }
+
+    @Override
+    public void connect() {
+        canalConnector.connect();
+        canalConnector.subscribe();
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
+            long batchId = message.getId();
+            currentBatchId = batchId;
+            int size = message.getEntries().size();
+            if (batchId == -1 || size == 0) {
+                return null;
+            } else {
+                return MessageUtil.convert(message);
+            }
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void rollback() {
+        if (currentBatchId != null && currentBatchId != -1) {
+            canalConnector.rollback(currentBatchId);
+            currentBatchId = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        if (currentBatchId != null) {
+            canalConnector.ack(currentBatchId);
+            currentBatchId = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        canalConnector.unsubscribe();
+        canalConnector.disconnect();
+    }
+}

connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+tcp=com.alibaba.otter.canal.connector.tcp.consumer.CanalTCPConsumer