chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed MQ partition

Browse Source
This commit is contained in:
agapple
2019-03-21 19:27:39 +08:00
parent 8fcdd7f798
commit a81d2710ff
36 changed files with 534 additions and 2850 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/AbstractKafkaTest.java
View File

@@ -1,26 +0,0 @@
package com.alibaba.otter.canal.client.running.kafka;
import org.junit.Assert;
/**
* Kafka 测试基类
*
* @author machengyuan @ 2018-6-12
* @version 1.0.0
*/
public abstract class AbstractKafkaTest {
public static String topic = "example";
public static Integer partition = null;
public static String groupId = "g4";
public static String servers = "slave1:6667,slave2:6667,slave3:6667";
public static String zkServers = "slave1:2181,slave2:2181,slave3:2181";
public void sleep(long time) {
try {
Thread.sleep(time);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
}
}

151
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java
View File

@@ -1,151 +0,0 @@
package com.alibaba.otter.canal.client.running.kafka;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka client example
*
* @author machengyuan @ 2018-6-12
* @version 1.0.0
*/
public class CanalKafkaClientExample {
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
private KafkaCanalConnector connector;
private static volatile boolean running = false;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
}
public static void main(String[] args) {
try {
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
AbstractKafkaTest.zkServers,
AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
}
});
while (running)
;
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the kafka consumer:", e);
System.exit(0);
}
}
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
public void stop() {
if (!running) {
return;
}
running = false;
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
}
private void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
connector.connect();
connector.subscribe();
while (running) {
try {
List<Message> messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); // 获取message
if (messages == null) {
continue;
}
for (Message message : messages) {
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
} else {
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
}
}
connector.ack(); // 提交确认
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
}
}

174
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaOffsetClientExample.java
View File

@@ -1,174 +0,0 @@
package com.alibaba.otter.canal.client.running.kafka;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.kafka.KafkaOffsetCanalConnector;
import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
/**
* KafkaOffsetCanalConnector 使用示例
* <p>KafkaOffsetCanalConnector 与 KafkaCanalConnector 的另一区别是 auto.offset.reset 默认值不同；</p>
* <p>KafkaOffsetCanalConnector 默认为 earliest；canal-kafka-client重启后从未被消费的记录开始拉取消息，同时提供了修改 auto.offset.reset 的方法 setAutoOffsetReset</p>
*
* @author panjianping @ 2018-12-18
* @version 1.1.3
*/
public class CanalKafkaOffsetClientExample {
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaOffsetClientExample.class);
private KafkaOffsetCanalConnector connector;
private static volatile boolean running = false;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaOffsetClientExample(String servers, String topic, Integer partition, String groupId) {
connector = new KafkaOffsetCanalConnector(servers, topic, partition, groupId, false);
}
public static void main(String[] args) {
try {
final CanalKafkaOffsetClientExample kafkaCanalClientExample = new CanalKafkaOffsetClientExample(
AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
}
});
while (running)
;
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the kafka consumer:", e);
System.exit(0);
}
}
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
public void stop() {
if (!running) {
return;
}
running = false;
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
}
private void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
// 修改 AutoOffsetReset 的值，默认（earliest）
//connector.setAutoOffsetReset(null);
connector.connect();
connector.subscribe();
// 消息起始偏移地址
long offset = -1;
// 错误次数
int errorCount = 0;
while (running) {
try {
// 错误重试次数超过3次后，每30秒递增重试
if (errorCount > 2) {
Thread.sleep((errorCount - 2) * 1000 * 30);
}
List<KafkaMessage> messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS, offset); // 获取message
if (messages == null) {
continue;
}
for (KafkaMessage message : messages) {
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
continue;
}
// 记录第一条消息的offset，用于处理数据异常时重新从此位置获取消息
if (offset < 0) {
offset = message.getOffset();
}
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
}
connector.ack(); // 提交确认
// 还原offset
offset = -1;
errorCount = 0;
} catch (Exception e) {
errorCount++;
logger.error(e.getMessage(), e);
if (errorCount == 3) {
// 重试3次后发送邮件提醒异常
// mailService.sendMail("同步数据异常，请及时处理", "错误消息");
}
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
}
}

62
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java
View File

@@ -1,62 +0,0 @@
package com.alibaba.otter.canal.client.running.kafka;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka consumer获取Message的测试例子
*
* @author machengyuan @ 2018-6-12
* @version 1.0.0
*/
public class KafkaClientRunningTest extends AbstractKafkaTest {
private Logger logger = LoggerFactory.getLogger(KafkaClientRunningTest.class);
private boolean running = true;
@Test
public void testKafkaConsumer() {
final ExecutorService executor = Executors.newFixedThreadPool(1);
final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
executor.submit(new Runnable() {
@Override
public void run() {
connector.connect();
connector.subscribe();
while (running) {
try {
List<Message> messages = connector.getList(3L, TimeUnit.SECONDS);
if (messages != null) {
System.out.println(messages);
}
connector.ack();
} catch (WakeupException e) {
// ignore
}
}
connector.unsubscribe();
connector.disconnect();
}
});
sleep(60000);
running = false;
executor.shutdown();
logger.info("shutdown completed");
}
}

9
client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/AbstractRocektMQTest.java
View File

@@ -1,9 +0,0 @@
package com.alibaba.otter.canal.client.running.rocketmq;
public class AbstractRocektMQTest {
public static String topic = "example";
public static String groupId = "group";
public static String nameServers = "localhost:9876";
}

142
client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java
View File

@@ -1,142 +0,0 @@
package com.alibaba.otter.canal.client.running.rocketmq;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.client.running.kafka.AbstractKafkaTest;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka client example
*
* @author machengyuan @ 2018-6-12
* @version 1.0.0
*/
public class CanalRocketMQClientExample extends AbstractRocektMQTest {
protected final static Logger logger = LoggerFactory.getLogger(CanalRocketMQClientExample.class);
private RocketMQCanalConnector connector;
private static volatile boolean running = false;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalRocketMQClientExample(String nameServers, String topic, String groupId){
connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false);
}
public static void main(String[] args) {
try {
final CanalRocketMQClientExample rocketMQClientExample = new CanalRocketMQClientExample(nameServers,
topic,
groupId);
logger.info("## Start the rocketmq consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
rocketMQClientExample.start();
logger.info("## The canal rocketmq consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## Stop the rocketmq consumer");
rocketMQClientExample.stop();
} catch (Throwable e) {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} finally {
logger.info("## Rocketmq consumer is down.");
}
}
});
while (running)
;
} catch (Throwable e) {
logger.error("## Something going wrong when starting up the rocketmq consumer:", e);
System.exit(0);
}
}
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
public void stop() {
if (!running) {
return;
}
running = false;
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
}
private void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
connector.connect();
connector.subscribe();
while (running) {
List<Message> messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); // 获取message
for (Message message : messages) {
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
} else {
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
}
}
connector.ack(); // 提交确认
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
// connector.stopRunning();
}
}