chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed mq

Browse Source
This commit is contained in:
agapple
2019-01-08 23:57:16 +08:00
parent b845a67666
commit 03e6662ce5
8 changed files with 87 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.client.rocketmq;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -26,6 +24,8 @@ import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import com.google.common.collect.Lists;
/**
@@ -48,24 +48,26 @@ public class RocketMQCanalConnector implements CanalMQConnector {
private volatile boolean connected = false;
private DefaultMQPushConsumer rocketMQConsumer;
private BlockingQueue<ConsumerBatchMessage> messageBlockingQueue;
private int batchSize = -1;
private long batchProcessTimeout = 60 * 1000;
private boolean flatMessage;
private volatile ConsumerBatchMessage lastGetBatchMessage = null;
private String accessKey;
private String secretKey;
public RocketMQCanalConnector(String nameServer, String topic, String groupName, boolean flatMessage){
public RocketMQCanalConnector(String nameServer, String topic, String groupName, Integer batchSize,
boolean flatMessage){
this.nameServer = nameServer;
this.topic = topic;
this.groupName = groupName;
this.flatMessage = flatMessage;
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
this.batchSize = batchSize;
}
public RocketMQCanalConnector(String nameServer, String topic, String groupName,
String accessKey, String secretKey, boolean flatMessage){
this(nameServer, topic, groupName, flatMessage);
public RocketMQCanalConnector(String nameServer, String topic, String groupName, String accessKey,
String secretKey, Integer batchSize, boolean flatMessage){
this(nameServer, topic, groupName, batchSize, flatMessage);
this.accessKey = accessKey;
this.secretKey = secretKey;
}
@@ -73,8 +75,7 @@ public class RocketMQCanalConnector implements CanalMQConnector {
public void connect() throws CanalClientException {
RPCHook rpcHook = null;
if(null != accessKey && accessKey.length() > 0
&& null != secretKey && secretKey.length() > 0){
if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(accessKey);
sessionCredentials.setSecretKey(secretKey);
@@ -85,6 +86,9 @@ public class RocketMQCanalConnector implements CanalMQConnector {
if (!StringUtils.isBlank(nameServer)) {
rocketMQConsumer.setNamesrvAddr(nameServer);
}
if (batchSize != -1) {
rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
}
}
public void disconnect() throws CanalClientException {

9
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java
View File

@@ -99,8 +99,13 @@ public class CanalKafkaClientExample {
}
private void process() {
while (!running)
;
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
connector.connect();

18
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java
View File

@@ -1,14 +1,15 @@
package com.alibaba.otter.canal.client.running.kafka;
import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector;
import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector;
import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
/**
* KafkaOffsetCanalConnector 使用示例
@@ -99,8 +100,13 @@ public class CanalKafkaOffsetClientExample {
}
private void process() {
while (!running)
;
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
// 修改 AutoOffsetReset 的值，默认（earliest）

11
client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java
View File

@@ -36,7 +36,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
};
public CanalRocketMQClientExample(String nameServers, String topic, String groupId){
connector = new RocketMQCanalConnector(nameServers, topic, groupId, false);
connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false);
}
public static void main(String[] args) {
@@ -97,8 +97,13 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
}
private void process() {
while (!running)
;
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
connector.connect();