fixed MQConnector

client/src/main/java/com/alibaba/otter/canal/client/CanalMQConnector.java

@@ -0,0 +1,91 @@
+package com.alibaba.otter.canal.client;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
+/**
+ * canal MQ数据操作客户端
+ * 
+ * <pre>
+ * 1. canal server写入MQ消息，考虑性能会合并多条数据写入为一个MQ消息，一个Message对应一个MQ消息
+ * 2. canal client消费MQ消息，因为client性能会弱于server的写入，MQ数据获取时会拿到堆积的多条MQ消息，会拿到List<Message>
+ * 3. client的ack/rollback，都是和MQ直接交互，不存在对应的batchId概念
+ * </pre>
+ * 
+ * @author agapple 2018年10月28日 下午6:42:27
+ * @since 1.1.1
+ */
+public interface CanalMQConnector extends CanalConnector {
+
+    /**
+     * 获取数据，自动进行确认，设置timeout时间直到拿到数据为止
+     * 
+     * <pre>
+     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     * </pre>
+     * 
+     * @return
+     * @throws CanalClientException
+     */
+    List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException;
+
+    /**
+     * 获取数据，设置timeout时间直到拿到数据为止
+     * 
+     * <pre>
+     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     * </pre>
+     * 
+     * @throws CanalClientException
+     */
+    List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException;
+
+    /**
+     * 获取数据，自动进行确认，设置timeout时间直到拿到数据为止
+     * 
+     * <pre>
+     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     * </pre>
+     * 
+     * @return
+     * @throws CanalClientException
+     */
+    List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException;
+
+    /**
+     * 获取数据，设置timeout时间直到拿到数据为止
+     * 
+     * <pre>
+     * 该方法返回的条件：
+     *  a. 如果timeout=0，有多少取多少，不会阻塞等待
+     *  b. 如果timeout不为0，尝试阻塞对应的超时时间，直到拿到数据就返回
+     * </pre>
+     * 
+     * @throws CanalClientException
+     */
+    List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException;
+
+    /**
+     * 消费确认。
+     * 
+     * @throws CanalClientException
+     */
+    void ack() throws CanalClientException;
+
+    /**
+     * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
+     * 
+     * @throws CanalClientException
+     */
+    void rollback() throws CanalClientException;
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -13,16 +13,25 @@ import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.StringDeserializer;
 
 import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.client.CanalMQConnector;
+import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
 
 /**
  * canal kafka 数据操作客户端
+ * 
+ * <pre>
+ * 注意点：
+ * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
+ * </pre>
  *
  * @author machengyuan @ 2018-6-12
- * @version 1.0.0
+ * @version 1.1.1
  */
-public class KafkaCanalConnector {
+public class KafkaCanalConnector implements CanalMQConnector {
 
     private KafkaConsumer<String, Message> kafkaConsumer;
     private KafkaConsumer<String, String>  kafkaConsumer2;   // 用于扁平message的数据消费
@@ -55,18 +64,6 @@ public class KafkaCanalConnector {
         }
     }
 
-    /**
-     * 重新设置sessionTime
-     *
-     * @param timeout
-     * @param unit
-     */
-    public void setSessionTimeout(Long timeout, TimeUnit unit) {
-        long t = unit.toMillis(timeout);
-        properties.put("request.timeout.ms", String.valueOf(t + 60000));
-        properties.put("session.timeout.ms", String.valueOf(t));
-    }
-
     /**
      * 打开连接
      */
@@ -76,7 +73,6 @@ public class KafkaCanalConnector {
         }
 
         connected = true;
-
         if (kafkaConsumer == null && !flatMessage) {
             kafkaConsumer = new KafkaConsumer<String, Message>(properties);
         }
@@ -151,62 +147,61 @@ public class KafkaCanalConnector {
         }
     }
 
-    /**
-     * 获取数据，自动进行确认
-     *
-     * @return
-     */
-    public List<Message> get() {
-        return get(100L, TimeUnit.MILLISECONDS);
-    }
-
-    public List<Message> get(Long timeout, TimeUnit unit) {
+    @Override
+    public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
         waitClientRunning();
         if (!running) {
-            return null;
+            return Lists.newArrayList();
         }
 
-        List<Message> messages = getWithoutAck(timeout, unit);
-        this.ack();
+        List<Message> messages = getListWithoutAck(timeout, unit);
+        if (messages != null && !messages.isEmpty()) {
+            this.ack();
+        }
         return messages;
     }
 
-    public List<Message> getWithoutAck() {
-        return getWithoutAck(100L, TimeUnit.MILLISECONDS);
-    }
-
-    /**
-     * 获取数据，不进行确认，等待处理完成手工确认
-     *
-     * @return
-     */
-    public List<Message> getWithoutAck(Long timeout, TimeUnit unit) {
+    @Override
+    public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
         waitClientRunning();
         if (!running) {
-            return null;
+            return Lists.newArrayList();
         }
 
         ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据
 
         if (!records.isEmpty()) {
-            // return records.iterator().next().value();
             List<Message> messages = new ArrayList<>();
             for (ConsumerRecord<String, Message> record : records) {
                 messages.add(record.value());
             }
             return messages;
         }
-        return null;
+        return Lists.newArrayList();
     }
 
-    public List<FlatMessage> getFlatMessageWithoutAck(Long timeout, TimeUnit unit) {
+    @Override
+    public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
         waitClientRunning();
         if (!running) {
-            return null;
+            return Lists.newArrayList();
+        }
+
+        List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
+        if (messages != null && !messages.isEmpty()) {
+            this.ack();
+        }
+        return messages;
+    }
+
+    @Override
+    public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
+        waitClientRunning();
+        if (!running) {
+            return Lists.newArrayList();
         }
 
         ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
-
         if (!records.isEmpty()) {
             List<FlatMessage> flatMessages = new ArrayList<>();
             for (ConsumerRecord<String, String> record : records) {
@@ -217,7 +212,11 @@ public class KafkaCanalConnector {
 
             return flatMessages;
         }
-        return null;
+        return Lists.newArrayList();
+    }
+
+    @Override
+    public void rollback() throws CanalClientException {
     }
 
     /**
@@ -245,4 +244,52 @@ public class KafkaCanalConnector {
             // }
         }
     }
+
+    @Override
+    public void subscribe(String filter) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message get(int batchSize) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message getWithoutAck(int batchSize) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public void ack(long batchId) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public void rollback(long batchId) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    /**
+     * 重新设置sessionTime
+     *
+     * @param timeout
+     * @param unit
+     */
+    public void setSessionTimeout(Long timeout, TimeUnit unit) {
+        long t = unit.toMillis(timeout);
+        properties.put("request.timeout.ms", String.valueOf(t + 60000));
+        properties.put("session.timeout.ms", String.valueOf(t));
+    }
+
 }

client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningData.java

@@ -1,39 +0,0 @@
-package com.alibaba.otter.canal.client.kafka.running;
-
-/**
- * client running状态信息
- *
- * @author machengyuan 2018-06-20 下午04:10:12
- * @version 1.0.0
- */
-public class ClientRunningData {
-
-    private String  groupId;
-    private String  address;
-    private boolean active = true;
-
-    public String getGroupId() {
-        return groupId;
-    }
-
-    public void setGroupId(String groupId) {
-        this.groupId = groupId;
-    }
-
-    public String getAddress() {
-        return address;
-    }
-
-    public void setAddress(String address) {
-        this.address = address;
-    }
-
-    public boolean isActive() {
-        return active;
-    }
-
-    public void setActive(boolean active) {
-        this.active = active;
-    }
-
-}

client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningListener.java

@@ -1,21 +0,0 @@
-package com.alibaba.otter.canal.client.kafka.running;
-
-/**
- * client running状态信息
- *
- * @author machengyuan 2018-06-20 下午04:10:12
- * @version 1.0.0
- */
-public interface ClientRunningListener {
-
-    /**
-     * 触发现在轮到自己做为active
-     */
-    public void processActiveEnter();
-
-    /**
-     * 触发一下当前active模式失败
-     */
-    public void processActiveExit();
-
-}

client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningMonitor.java

@@ -1,280 +0,0 @@
-package com.alibaba.otter.canal.client.kafka.running;
-
-import java.text.MessageFormat;
-import java.util.Random;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.I0Itec.zkclient.IZkDataListener;
-import org.I0Itec.zkclient.exception.ZkException;
-import org.I0Itec.zkclient.exception.ZkInterruptedException;
-import org.I0Itec.zkclient.exception.ZkNoNodeException;
-import org.I0Itec.zkclient.exception.ZkNodeExistsException;
-import org.apache.zookeeper.CreateMode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
-import com.alibaba.otter.canal.common.utils.AddressUtils;
-import com.alibaba.otter.canal.common.utils.BooleanMutex;
-import com.alibaba.otter.canal.common.utils.JsonUtils;
-import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
-import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
-import com.alibaba.otter.canal.protocol.exception.CanalClientException;
-
-/**
- * kafka client running状态信息
- *
- * @author machengyuan 2018-06-20 下午04:10:12
- * @version 1.0.0
- */
-public class ClientRunningMonitor extends AbstractCanalLifeCycle {
-
-    private static final String TOPIC_ROOT_NODE             = ZookeeperPathUtils.CANAL_ROOT_NODE
-                                                              + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics";
-
-    private static final String TOPIC_NODE                  = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
-                                                              + "{0}";
-
-    private static final String TOPIC_CLIENTID_NODE         = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
-                                                              + "{1}";
-
-    private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE
-                                                              + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
-                                                              + ZookeeperPathUtils.RUNNING_NODE;
-
-    private static String getTopicClientRunning(String topic, String groupId) {
-        return MessageFormat.format(TOPIC_CLIENTID_RUNNING_NODE, topic, groupId);
-    }
-
-    private static String getClientIdNodePath(String topic, String groupId) {
-        return MessageFormat.format(TOPIC_CLIENTID_NODE, topic, groupId);
-    }
-
-    private static final Logger        logger       = LoggerFactory.getLogger(ClientRunningMonitor.class);
-    private ZkClientx                  zkClient;
-    private String                     topic;
-    private ClientRunningData          clientData;
-    private IZkDataListener            dataListener;
-    private BooleanMutex               mutex        = new BooleanMutex(false);
-    private volatile boolean           release      = false;
-    private volatile ClientRunningData activeData;
-    private ScheduledExecutorService   delayExector = Executors.newScheduledThreadPool(1);
-    private ClientRunningListener      listener;
-    private int                        delayTime    = 5;
-
-    private static Integer             virtualPort;
-
-    public ClientRunningMonitor(){
-        if (virtualPort == null) {
-            Random rand = new Random();
-            virtualPort = rand.nextInt(9000) + 1000;
-        }
-
-        dataListener = new IZkDataListener() {
-
-            public void handleDataChange(String dataPath, Object data) throws Exception {
-                MDC.put("kafkaTopic", topic);
-                ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class);
-                if (!isMine(runningData.getAddress())) {
-                    mutex.set(false);
-                }
-
-                if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
-                    release = true;
-                    releaseRunning();// 彻底释放mainstem
-                }
-
-                activeData = (ClientRunningData) runningData;
-            }
-
-            public void handleDataDeleted(String dataPath) throws Exception {
-                MDC.put("kafkaTopic", topic);
-
-                mutex.set(false);
-                // 触发一下退出,可能是人为干预的释放操作或者网络闪断引起的session expired timeout
-                processActiveExit();
-                if (!release && activeData != null && isMine(activeData.getAddress())) {
-                    // 如果上一次active的状态就是本机，则即时触发一下active抢占
-                    initRunning();
-                } else {
-                    // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
-                    delayExector.schedule(new Runnable() {
-
-                        public void run() {
-                            initRunning();
-                        }
-                    }, delayTime, TimeUnit.SECONDS);
-                }
-            }
-
-        };
-
-    }
-
-    public void start() {
-        super.start();
-
-        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
-
-        zkClient.subscribeDataChanges(path, dataListener);
-        initRunning();
-    }
-
-    public void stop() {
-        super.stop();
-        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
-        zkClient.unsubscribeDataChanges(path, dataListener);
-        releaseRunning(); // 尝试一下release
-        // Fix issue #697
-        if (delayExector != null) {
-            delayExector.shutdown();
-        }
-    }
-
-    // 改动记录：
-    // 1,在方法上加synchronized关键字,保证同步顺序执行;
-    // 2,判断Zk上已经存在的activeData是否是本机，是的话把mutex重置为true，否则会导致死锁
-    // 3,增加异常处理，保证出现异常时，running节点能被删除,否则会导致死锁
-    public synchronized void initRunning() {
-        if (!isStart()) {
-            return;
-        }
-
-        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
-        // 序列化
-        byte[] bytes = JsonUtils.marshalToByte(clientData);
-        try {
-            mutex.set(false);
-            zkClient.create(path, bytes, CreateMode.EPHEMERAL);
-            processActiveEnter();// 触发一下事件
-            activeData = clientData;
-            mutex.set(true);
-        } catch (ZkNodeExistsException e) {
-            bytes = zkClient.readData(path, true);
-            if (bytes == null) {// 如果不存在节点，立即尝试一次
-                initRunning();
-            } else {
-                activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
-                // 如果发现已经存在,判断一下是否自己,避免活锁
-                if (activeData.getAddress().contains(":") && isMine(activeData.getAddress())) {
-                    mutex.set(true);
-                }
-            }
-        } catch (ZkNoNodeException e) {
-            zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), true); // 尝试创建父节点
-            initRunning();
-        } catch (Throwable t) {
-            logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].",
-                topic),
-                t);
-            // 出现任何异常尝试release
-            releaseRunning();
-            throw new CanalClientException("something goes wrong in initRunning method. ", t);
-        }
-    }
-
-    /**
-     * 阻塞等待自己成为active，如果自己成为active，立马返回
-     *
-     * @throws InterruptedException
-     */
-    public void waitForActive() throws InterruptedException {
-        initRunning();
-        mutex.get();
-    }
-
-    /**
-     * 检查当前的状态
-     */
-    public boolean check() {
-        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
-        // ZookeeperPathUtils.getDestinationClientRunning(this.destination,
-        // clientData.getClientId());
-        try {
-            byte[] bytes = zkClient.readData(path);
-            ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
-            activeData = eventData;// 更新下为最新值
-            // 检查下nid是否为自己
-            boolean result = isMine(activeData.getAddress());
-            if (!result) {
-                logger.warn("canal is running in [{}] , but not in [{}]",
-                    activeData.getAddress(),
-                    clientData.getAddress());
-            }
-            return result;
-        } catch (ZkNoNodeException e) {
-            logger.warn("canal is not run any in node");
-            return false;
-        } catch (ZkInterruptedException e) {
-            logger.warn("canal check is interrupt");
-            Thread.interrupted();// 清除interrupt标记
-            return check();
-        } catch (ZkException e) {
-            logger.warn("canal check is failed");
-            return false;
-        }
-    }
-
-    public boolean releaseRunning() {
-        if (check()) {
-            String path = getTopicClientRunning(this.topic, clientData.getGroupId());
-            zkClient.delete(path);
-            mutex.set(false);
-            processActiveExit();
-            return true;
-        }
-
-        return false;
-    }
-
-    // ====================== helper method ======================
-
-    private boolean isMine(String address) {
-        return address.equals(clientData.getAddress());
-    }
-
-    private void processActiveEnter() {
-        if (listener != null) {
-            // 触发回调
-            listener.processActiveEnter();
-            this.clientData.setAddress(/* address */AddressUtils.getHostIp() + ":" + virtualPort);
-
-            String path = getTopicClientRunning(this.topic, clientData.getGroupId());
-            // 序列化
-            byte[] bytes = JsonUtils.marshalToByte(clientData);
-            zkClient.writeData(path, bytes);
-        }
-    }
-
-    private void processActiveExit() {
-        if (listener != null) {
-            listener.processActiveExit();
-        }
-    }
-
-    public void setListener(ClientRunningListener listener) {
-        this.listener = listener;
-    }
-
-    // ===================== setter / getter =======================
-
-    public void setTopic(String topic) {
-        this.topic = topic;
-    }
-
-    public void setClientData(ClientRunningData clientData) {
-        this.clientData = clientData;
-    }
-
-    public void setDelayTime(int delayTime) {
-        this.delayTime = delayTime;
-    }
-
-    public void setZkClient(ZkClientx zkClient) {
-        this.zkClient = zkClient;
-    }
-
-}

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/ConsumerBatchMessage.java

@@ -1,18 +1,18 @@
 package com.alibaba.otter.canal.client.rocketmq;
 
-import java.util.concurrent.BlockingQueue;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 public class ConsumerBatchMessage<T> {
 
-    private final BlockingQueue<T> data;
+    private final List<T>  data;
     private CountDownLatch         latch;
     private boolean                hasFailure = false;
 
-    public ConsumerBatchMessage(BlockingQueue<T> data){
+    public ConsumerBatchMessage(List<T> data){
         this.data = data;
-        latch = new CountDownLatch(data.size());
+        latch = new CountDownLatch(1);
     }
 
     public boolean waitFinish(long timeout) throws InterruptedException {
@@ -23,7 +23,7 @@ public class ConsumerBatchMessage<T> {
         return !hasFailure;
     }
 
-    public BlockingQueue<T> getData() {
+    public List<T> getData() {
         return data;
     }
 

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java

@@ -1,11 +1,7 @@
 package com.alibaba.otter.canal.client.rocketmq;
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.otter.canal.protocol.FlatMessage;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
@@ -19,35 +15,47 @@ import org.apache.rocketmq.common.message.MessageExt;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.client.CanalMQConnector;
 import com.alibaba.otter.canal.client.CanalMessageDeserializer;
+import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
 
-public class RocketMQCanalConnector implements CanalConnector {
+/**
+ * RocketMQ的连接
+ * 
+ * <pre>
+ * 注意点:
+ * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
+ * </pre>
+ * 
+ * @since 1.1.1
+ */
+public class RocketMQCanalConnector implements CanalMQConnector {
 
-    private static final Logger logger = LoggerFactory.getLogger(RocketMQCanalConnector.class);
+    private static final Logger                 logger              = LoggerFactory.getLogger(RocketMQCanalConnector.class);
 
-    private String nameServer;
-    private String topic;
-    private String groupName;
-    private volatile boolean connected = false;
-    private DefaultMQPushConsumer rocketMQConsumer;
+    private String                              nameServer;
+    private String                              topic;
+    private String                              groupName;
+    private volatile boolean                    connected           = false;
+    private DefaultMQPushConsumer               rocketMQConsumer;
     private BlockingQueue<ConsumerBatchMessage> messageBlockingQueue;
-    Map<Long, ConsumerBatchMessage> messageCache;
-    private long batchProcessTimeout = 3000;
-    private boolean flatMessage;
+    private long                                batchProcessTimeout = 60 * 1000;
+    private boolean                             flatMessage;
+    private volatile ConsumerBatchMessage       lastGetBatchMessage = null;
 
-    public RocketMQCanalConnector(String nameServer, String topic, String groupName, boolean flatMessage) {
+    public RocketMQCanalConnector(String nameServer, String topic, String groupName, boolean flatMessage){
         this.nameServer = nameServer;
         this.topic = topic;
         this.groupName = groupName;
         this.flatMessage = flatMessage;
-        messageBlockingQueue = new LinkedBlockingQueue<>();
-        messageCache = new ConcurrentHashMap<>();
+        this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
     }
 
-    @Override
     public void connect() throws CanalClientException {
         rocketMQConsumer = new DefaultMQPushConsumer(groupName);
         if (!StringUtils.isBlank(nameServer)) {
@@ -55,17 +63,14 @@ public class RocketMQCanalConnector implements CanalConnector {
         }
     }
 
-    @Override
     public void disconnect() throws CanalClientException {
         rocketMQConsumer.shutdown();
     }
 
-    @Override
     public boolean checkValid() throws CanalClientException {
         return connected;
     }
 
-    @Override
     public synchronized void subscribe(String filter) throws CanalClientException {
         if (connected) {
             return;
@@ -78,8 +83,7 @@ public class RocketMQCanalConnector implements CanalConnector {
             rocketMQConsumer.registerMessageListener(new MessageListenerOrderly() {
 
                 @Override
-                public ConsumeOrderlyStatus consumeMessage(List<MessageExt> messageExts,
-                    ConsumeOrderlyContext context) {
+                public ConsumeOrderlyStatus consumeMessage(List<MessageExt> messageExts, ConsumeOrderlyContext context) {
                     context.setAutoCommit(true);
                     boolean isSuccess = process(messageExts);
                     if (isSuccess) {
@@ -99,23 +103,23 @@ public class RocketMQCanalConnector implements CanalConnector {
 
     private boolean process(List<MessageExt> messageExts) {
         logger.info("Get Message:{}", messageExts);
-        BlockingQueue messageList = new LinkedBlockingQueue<>();
+        List messageList = Lists.newArrayList();
         for (MessageExt messageExt : messageExts) {
             byte[] data = messageExt.getBody();
-            if (data != null){
+            if (data != null) {
                 try {
                     if (!flatMessage) {
                         Message message = CanalMessageDeserializer.deserializer(data);
-                        messageList.put(message);
+                        messageList.add(message);
                     } else {
                         FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class);
-                        messageList.put(flatMessage);
+                        messageList.add(flatMessage);
                     }
                 } catch (Exception ex) {
                     logger.error("Add message error", ex);
                     throw new CanalClientException(ex);
                 }
-            }else{
+            } else {
                 logger.warn("Received message data is null");
             }
         }
@@ -142,145 +146,117 @@ public class RocketMQCanalConnector implements CanalConnector {
         return isCompleted && isSuccess;
     }
 
-    @Override
     public void subscribe() throws CanalClientException {
         this.subscribe(null);
     }
 
-    @Override
     public void unsubscribe() throws CanalClientException {
         this.rocketMQConsumer.unsubscribe(this.topic);
     }
 
-    /**
-     * 暂时不支持batchSize 参数
-     *
-     * @param batchSize 暂时不支持
-     * @return
-     * @throws CanalClientException
-     */
     @Override
-    public Message get(int batchSize) throws CanalClientException {
-        Message message = getWithoutAck(batchSize);
-        if (message != null) {
-            ack(message.getId());
+    public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
+        List<Message> messages = getListWithoutAck(timeout, unit);
+        if (messages != null && !messages.isEmpty()) {
+            ack();
         }
-        return message;
+        return messages;
     }
 
     @Override
-    public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
-        Message message = getWithoutAck(batchSize, timeout, unit);
-        if (message != null) {
-            ack(message.getId());
-        }
-        return message;
-    }
-
-    private Message getMessage(ConsumerBatchMessage consumerBatchMessage) {
-        BlockingQueue<Message> messageList = consumerBatchMessage.getData();
-        if (messageList != null & messageList.size() > 0) {
-            Message message = messageList.poll();
-            messageCache.put(message.getId(), consumerBatchMessage);
-            return message;
-        }
-        return null;
-    }
-
-    private FlatMessage getFlatMessage(ConsumerBatchMessage consumerBatchMessage) {
-        BlockingQueue<FlatMessage> messageList = consumerBatchMessage.getData();
-        if (messageList != null & messageList.size() > 0) {
-            FlatMessage message = messageList.poll();
-            messageCache.put(message.getId(), consumerBatchMessage);
-            return message;
-        }
-        return null;
-    }
-
-    /**
-     * 暂时不支持该参数设置
-     *
-     * @param batchSize
-     * @return
-     * @throws CanalClientException
-     */
-    @Override
-    public Message getWithoutAck(int batchSize) throws CanalClientException {
-        ConsumerBatchMessage batchMessage = messageBlockingQueue.poll();
-        if (batchMessage != null) {
-            return getMessage(batchMessage);
-        }
-        return null;
-    }
-
-    @Override
-    public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+    public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
         try {
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
+            }
+
             ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit);
             if (batchMessage != null) {
-                return getMessage(batchMessage);
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
             }
         } catch (InterruptedException ex) {
             logger.warn("Get message timeout", ex);
             throw new CanalClientException("Failed to fetch the data after: " + timeout);
         }
-        return null;
+        return Lists.newArrayList();
     }
 
-    public FlatMessage getFlatMessageWithoutAck() {
-        return getFlatMessageWithoutAck(null, null);
+    @Override
+    public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
+        List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
+        if (messages != null && !messages.isEmpty()) {
+            ack();
+        }
+        return messages;
     }
 
-    public FlatMessage getFlatMessageWithoutAck(Long timeout,
-        TimeUnit unit) throws CanalClientException {
+    @Override
+    public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
         try {
-            ConsumerBatchMessage batchMessage = null;
-            if (timeout == null || timeout == 0) {
-                batchMessage = messageBlockingQueue.poll();
-            } else {
-                batchMessage = messageBlockingQueue.poll(timeout, unit);
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
             }
+
+            ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit);
             if (batchMessage != null) {
-                return getFlatMessage(batchMessage);
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
             }
         } catch (InterruptedException ex) {
-            logger.warn("Get flat message timeout", ex);
-            throw new CanalClientException("Failed to fetch the flat message data after: " + timeout);
+            logger.warn("Get message timeout", ex);
+            throw new CanalClientException("Failed to fetch the data after: " + timeout);
         }
-        return null;
-    }
-
-    public FlatMessage getFlatMessage() throws CanalClientException {
-        FlatMessage message = getFlatMessageWithoutAck(null, null);
-        if (message != null) {
-            ack(message.getId());
-        }
-        return message;
-    }
-
-    public FlatMessage getFlatMessage(Long timeout, TimeUnit unit) throws CanalClientException {
-        FlatMessage message = getFlatMessageWithoutAck(timeout, unit);
-        ack(message.getId());
-        return message;
+        return Lists.newArrayList();
     }
 
     @Override
-    public void ack(long batchId) throws CanalClientException {
-        ConsumerBatchMessage batchMessage = messageCache.get(batchId);
-        if (batchMessage != null) {
-            batchMessage.ack();
-            messageCache.remove(batchId);
+    public void ack() throws CanalClientException {
+        try {
+            this.lastGetBatchMessage.ack();
+        } catch (Throwable e) {
+            this.lastGetBatchMessage.fail();
+        } finally {
+            this.lastGetBatchMessage = null;
         }
     }
 
-    @Override
-    public void rollback(long batchId) throws CanalClientException {
-
-    }
-
     @Override
     public void rollback() throws CanalClientException {
+        try {
+            this.lastGetBatchMessage.fail();
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
 
+    public Message get(int batchSize) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message getWithoutAck(int batchSize) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public void ack(long batchId) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public void rollback(long batchId) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
     }
 
     @Override

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java

@@ -3,7 +3,7 @@ package com.alibaba.otter.canal.client.rocketmq;
 /**
  * RocketMQ connector provider.
  */
-public class RocketMQCanalConnectorProvider {
+public class RocketMQCanalConnectors {
 
     /**
      * Create RocketMQ connector
@@ -18,7 +18,7 @@ public class RocketMQCanalConnectorProvider {
     }
 
     public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId,
-        boolean flatMessage) {
+                                                              boolean flatMessage) {
         return new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage);
     }
 }

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalListener.java

@@ -1,13 +0,0 @@
-package com.alibaba.otter.canal.client.rocketmq;
-
-import java.util.List;
-
-import org.apache.rocketmq.common.message.MessageExt;
-
-/**
- * RocketMQ message listener
- */
-public interface RocketMQCanalListener {
-
-    boolean onReceive(List<MessageExt> messageExts);
-}