mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
kafka consumer connector引入事务隔离级别 fix #1512
This commit is contained in:
mcy
@@ -1,325 +1,326 @@
|
|||||||
package com.alibaba.otter.canal.client.kafka;
|
package com.alibaba.otter.canal.client.kafka;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||||
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
||||||
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
||||||
import org.apache.kafka.common.TopicPartition;
|
import org.apache.kafka.common.TopicPartition;
|
||||||
import org.apache.kafka.common.serialization.StringDeserializer;
|
import org.apache.kafka.common.serialization.StringDeserializer;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.alibaba.otter.canal.client.CanalMQConnector;
|
import com.alibaba.otter.canal.client.CanalMQConnector;
|
||||||
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
|
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
|
||||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||||
import com.alibaba.otter.canal.protocol.Message;
|
import com.alibaba.otter.canal.protocol.Message;
|
||||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* canal kafka 数据操作客户端
|
* canal kafka 数据操作客户端
|
||||||
*
|
*
|
||||||
* <pre>
|
* <pre>
|
||||||
* 注意点：
|
* 注意点：
|
||||||
* 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
|
* 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
* @author machengyuan @ 2018-6-12
|
* @author machengyuan @ 2018-6-12
|
||||||
* @version 1.1.1
|
* @version 1.1.1
|
||||||
*/
|
*/
|
||||||
public class KafkaCanalConnector implements CanalMQConnector {
|
public class KafkaCanalConnector implements CanalMQConnector {
|
||||||
|
|
||||||
protected KafkaConsumer<String, Message> kafkaConsumer;
|
protected KafkaConsumer<String, Message> kafkaConsumer;
|
||||||
protected KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费
|
protected KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费
|
||||||
protected String topic;
|
protected String topic;
|
||||||
protected Integer partition;
|
protected Integer partition;
|
||||||
protected Properties properties;
|
protected Properties properties;
|
||||||
protected volatile boolean connected = false;
|
protected volatile boolean connected = false;
|
||||||
protected volatile boolean running = false;
|
protected volatile boolean running = false;
|
||||||
protected boolean flatMessage;
|
protected boolean flatMessage;
|
||||||
|
|
||||||
private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
|
private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
|
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
|
||||||
boolean flatMessage){
|
boolean flatMessage){
|
||||||
this.topic = topic;
|
this.topic = topic;
|
||||||
this.partition = partition;
|
this.partition = partition;
|
||||||
this.flatMessage = flatMessage;
|
this.flatMessage = flatMessage;
|
||||||
|
|
||||||
properties = new Properties();
|
properties = new Properties();
|
||||||
properties.put("bootstrap.servers", servers);
|
properties.put("bootstrap.servers", servers);
|
||||||
properties.put("group.id", groupId);
|
properties.put("group.id", groupId);
|
||||||
properties.put("enable.auto.commit", false);
|
properties.put("enable.auto.commit", false);
|
||||||
properties.put("auto.commit.interval.ms", "1000");
|
properties.put("auto.commit.interval.ms", "1000");
|
||||||
properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
|
properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
|
||||||
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
|
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
|
||||||
properties.put("session.timeout.ms", "30000"); // 默认为30秒
|
properties.put("session.timeout.ms", "30000"); // 默认为30秒
|
||||||
if (batchSize == null) {
|
properties.put("isolation.level", "read_committed");
|
||||||
batchSize = 100;
|
if (batchSize == null) {
|
||||||
}
|
batchSize = 100;
|
||||||
properties.put("max.poll.records", batchSize.toString());
|
}
|
||||||
properties.put("key.deserializer", StringDeserializer.class.getName());
|
properties.put("max.poll.records", batchSize.toString());
|
||||||
if (!flatMessage) {
|
properties.put("key.deserializer", StringDeserializer.class.getName());
|
||||||
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
if (!flatMessage) {
|
||||||
} else {
|
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
||||||
properties.put("value.deserializer", StringDeserializer.class.getName());
|
} else {
|
||||||
}
|
properties.put("value.deserializer", StringDeserializer.class.getName());
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 打开连接
|
/**
|
||||||
*/
|
* 打开连接
|
||||||
@Override
|
*/
|
||||||
public void connect() {
|
@Override
|
||||||
if (connected) {
|
public void connect() {
|
||||||
return;
|
if (connected) {
|
||||||
}
|
return;
|
||||||
|
}
|
||||||
connected = true;
|
|
||||||
if (kafkaConsumer == null && !flatMessage) {
|
connected = true;
|
||||||
kafkaConsumer = new KafkaConsumer<String, Message>(properties);
|
if (kafkaConsumer == null && !flatMessage) {
|
||||||
|
kafkaConsumer = new KafkaConsumer<String, Message>(properties);
|
||||||
}
|
|
||||||
if (kafkaConsumer2 == null && flatMessage) {
|
}
|
||||||
kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
|
if (kafkaConsumer2 == null && flatMessage) {
|
||||||
}
|
kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 关闭链接
|
/**
|
||||||
*/
|
* 关闭链接
|
||||||
@Override
|
*/
|
||||||
public void disconnect() {
|
@Override
|
||||||
if (kafkaConsumer != null) {
|
public void disconnect() {
|
||||||
kafkaConsumer.close();
|
if (kafkaConsumer != null) {
|
||||||
kafkaConsumer = null;
|
kafkaConsumer.close();
|
||||||
}
|
kafkaConsumer = null;
|
||||||
if (kafkaConsumer2 != null) {
|
}
|
||||||
kafkaConsumer2.close();
|
if (kafkaConsumer2 != null) {
|
||||||
kafkaConsumer2 = null;
|
kafkaConsumer2.close();
|
||||||
}
|
kafkaConsumer2 = null;
|
||||||
|
}
|
||||||
connected = false;
|
|
||||||
}
|
connected = false;
|
||||||
|
}
|
||||||
protected void waitClientRunning() {
|
|
||||||
running = true;
|
protected void waitClientRunning() {
|
||||||
}
|
running = true;
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public boolean checkValid() {
|
@Override
|
||||||
return true;// 默认都放过
|
public boolean checkValid() {
|
||||||
}
|
return true;// 默认都放过
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 订阅topic
|
/**
|
||||||
*/
|
* 订阅topic
|
||||||
@Override
|
*/
|
||||||
public void subscribe() {
|
@Override
|
||||||
waitClientRunning();
|
public void subscribe() {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return;
|
if (!running) {
|
||||||
}
|
return;
|
||||||
|
}
|
||||||
if (partition == null) {
|
|
||||||
if (kafkaConsumer != null) {
|
if (partition == null) {
|
||||||
kafkaConsumer.subscribe(Collections.singletonList(topic));
|
if (kafkaConsumer != null) {
|
||||||
}
|
kafkaConsumer.subscribe(Collections.singletonList(topic));
|
||||||
if (kafkaConsumer2 != null) {
|
}
|
||||||
kafkaConsumer2.subscribe(Collections.singletonList(topic));
|
if (kafkaConsumer2 != null) {
|
||||||
}
|
kafkaConsumer2.subscribe(Collections.singletonList(topic));
|
||||||
} else {
|
}
|
||||||
TopicPartition topicPartition = new TopicPartition(topic, partition);
|
} else {
|
||||||
if (kafkaConsumer != null) {
|
TopicPartition topicPartition = new TopicPartition(topic, partition);
|
||||||
kafkaConsumer.assign(Collections.singletonList(topicPartition));
|
if (kafkaConsumer != null) {
|
||||||
}
|
kafkaConsumer.assign(Collections.singletonList(topicPartition));
|
||||||
if (kafkaConsumer2 != null) {
|
}
|
||||||
kafkaConsumer2.assign(Collections.singletonList(topicPartition));
|
if (kafkaConsumer2 != null) {
|
||||||
}
|
kafkaConsumer2.assign(Collections.singletonList(topicPartition));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 取消订阅
|
/**
|
||||||
*/
|
* 取消订阅
|
||||||
@Override
|
*/
|
||||||
public void unsubscribe() {
|
@Override
|
||||||
waitClientRunning();
|
public void unsubscribe() {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return;
|
if (!running) {
|
||||||
}
|
return;
|
||||||
|
}
|
||||||
if (kafkaConsumer != null) {
|
|
||||||
kafkaConsumer.unsubscribe();
|
if (kafkaConsumer != null) {
|
||||||
}
|
kafkaConsumer.unsubscribe();
|
||||||
if (kafkaConsumer2 != null) {
|
}
|
||||||
kafkaConsumer2.unsubscribe();
|
if (kafkaConsumer2 != null) {
|
||||||
}
|
kafkaConsumer2.unsubscribe();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
|
@Override
|
||||||
waitClientRunning();
|
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return Lists.newArrayList();
|
if (!running) {
|
||||||
}
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
List<Message> messages = getListWithoutAck(timeout, unit);
|
|
||||||
if (messages != null && !messages.isEmpty()) {
|
List<Message> messages = getListWithoutAck(timeout, unit);
|
||||||
this.ack();
|
if (messages != null && !messages.isEmpty()) {
|
||||||
}
|
this.ack();
|
||||||
return messages;
|
}
|
||||||
}
|
return messages;
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
|
@Override
|
||||||
waitClientRunning();
|
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return Lists.newArrayList();
|
if (!running) {
|
||||||
}
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
|
|
||||||
|
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
|
||||||
currentOffsets.clear();
|
|
||||||
for (TopicPartition topicPartition : records.partitions()) {
|
currentOffsets.clear();
|
||||||
currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
|
for (TopicPartition topicPartition : records.partitions()) {
|
||||||
}
|
currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
|
||||||
|
}
|
||||||
if (!records.isEmpty()) {
|
|
||||||
List<Message> messages = new ArrayList<>();
|
if (!records.isEmpty()) {
|
||||||
for (ConsumerRecord<String, Message> record : records) {
|
List<Message> messages = new ArrayList<>();
|
||||||
messages.add(record.value());
|
for (ConsumerRecord<String, Message> record : records) {
|
||||||
}
|
messages.add(record.value());
|
||||||
return messages;
|
}
|
||||||
}
|
return messages;
|
||||||
return Lists.newArrayList();
|
}
|
||||||
}
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
|
@Override
|
||||||
waitClientRunning();
|
public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return Lists.newArrayList();
|
if (!running) {
|
||||||
}
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
|
|
||||||
if (messages != null && !messages.isEmpty()) {
|
List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
|
||||||
this.ack();
|
if (messages != null && !messages.isEmpty()) {
|
||||||
}
|
this.ack();
|
||||||
return messages;
|
}
|
||||||
}
|
return messages;
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
|
@Override
|
||||||
waitClientRunning();
|
public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return Lists.newArrayList();
|
if (!running) {
|
||||||
}
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
|
|
||||||
|
ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
|
||||||
currentOffsets.clear();
|
|
||||||
for (TopicPartition topicPartition : records.partitions()) {
|
currentOffsets.clear();
|
||||||
currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
|
for (TopicPartition topicPartition : records.partitions()) {
|
||||||
}
|
currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
|
||||||
|
}
|
||||||
if (!records.isEmpty()) {
|
|
||||||
List<FlatMessage> flatMessages = new ArrayList<>();
|
if (!records.isEmpty()) {
|
||||||
for (ConsumerRecord<String, String> record : records) {
|
List<FlatMessage> flatMessages = new ArrayList<>();
|
||||||
String flatMessageJson = record.value();
|
for (ConsumerRecord<String, String> record : records) {
|
||||||
FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
|
String flatMessageJson = record.value();
|
||||||
flatMessages.add(flatMessage);
|
FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
|
||||||
}
|
flatMessages.add(flatMessage);
|
||||||
|
}
|
||||||
return flatMessages;
|
|
||||||
}
|
return flatMessages;
|
||||||
return Lists.newArrayList();
|
}
|
||||||
}
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public void rollback() {
|
@Override
|
||||||
waitClientRunning();
|
public void rollback() {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return;
|
if (!running) {
|
||||||
}
|
return;
|
||||||
// 回滚所有分区
|
}
|
||||||
if (kafkaConsumer != null) {
|
// 回滚所有分区
|
||||||
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
|
if (kafkaConsumer != null) {
|
||||||
kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
|
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
|
||||||
}
|
kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
|
||||||
}
|
}
|
||||||
if (kafkaConsumer2 != null) {
|
}
|
||||||
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
|
if (kafkaConsumer2 != null) {
|
||||||
kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
|
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
|
||||||
}
|
kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败
|
/**
|
||||||
*/
|
* 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败
|
||||||
@Override
|
*/
|
||||||
public void ack() {
|
@Override
|
||||||
waitClientRunning();
|
public void ack() {
|
||||||
if (!running) {
|
waitClientRunning();
|
||||||
return;
|
if (!running) {
|
||||||
}
|
return;
|
||||||
|
}
|
||||||
if (kafkaConsumer != null) {
|
|
||||||
kafkaConsumer.commitSync();
|
if (kafkaConsumer != null) {
|
||||||
}
|
kafkaConsumer.commitSync();
|
||||||
if (kafkaConsumer2 != null) {
|
}
|
||||||
kafkaConsumer2.commitSync();
|
if (kafkaConsumer2 != null) {
|
||||||
}
|
kafkaConsumer2.commitSync();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public void subscribe(String filter) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public void subscribe(String filter) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Message get(int batchSize) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public Message get(int batchSize) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Message getWithoutAck(int batchSize) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public Message getWithoutAck(int batchSize) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public void ack(long batchId) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public void ack(long batchId) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public void rollback(long batchId) throws CanalClientException {
|
@Override
|
||||||
throw new CanalClientException("mq not support this method");
|
public void rollback(long batchId) throws CanalClientException {
|
||||||
}
|
throw new CanalClientException("mq not support this method");
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 重新设置sessionTime
|
/**
|
||||||
*
|
* 重新设置sessionTime
|
||||||
* @param timeout
|
*
|
||||||
* @param unit
|
* @param timeout
|
||||||
*/
|
* @param unit
|
||||||
public void setSessionTimeout(Long timeout, TimeUnit unit) {
|
*/
|
||||||
long t = unit.toMillis(timeout);
|
public void setSessionTimeout(Long timeout, TimeUnit unit) {
|
||||||
properties.put("request.timeout.ms", String.valueOf(t + 60000));
|
long t = unit.toMillis(timeout);
|
||||||
properties.put("session.timeout.ms", String.valueOf(t));
|
properties.put("request.timeout.ms", String.valueOf(t + 60000));
|
||||||
}
|
properties.put("session.timeout.ms", String.valueOf(t));
|
||||||
|
}
|
||||||
}
|
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user