diff --git a/deployer/src/main/resources/kafka.yml b/deployer/src/main/resources/kafka.yml index 8f897e6d..edd6b464 100644 --- a/deployer/src/main/resources/kafka.yml +++ b/deployer/src/main/resources/kafka.yml @@ -3,7 +3,11 @@ retries: 0 batchSize: 16384 lingerMs: 1 bufferMemory: 33554432 -canalBatchSize: 50 # Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下) + +# Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下) +canalBatchSize: 50 +# Canal get数据的超时时间, 单位: 毫秒, 空为不限超时 +canalGetTimeout: 100 canalDestinations: - canalDestination: example diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java index 4ad17e25..717a4525 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaStarter.java @@ -4,6 +4,7 @@ import java.io.FileInputStream; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -117,7 +118,16 @@ public class CanalKafkaStarter implements CanalServerStarter { logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination()); while (running) { - Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据 + Message message; + if (kafkaProperties.getCanalGetTimeout() != null) { + message = server.getWithoutAck(clientIdentity, + kafkaProperties.getCanalBatchSize(), + kafkaProperties.getCanalGetTimeout(), + TimeUnit.MILLISECONDS); + } else { + message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); + } + final long batchId = message.getId(); try { int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java b/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java index 9194c0ab..40f11bf5 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java @@ -20,6 +20,7 @@ public class KafkaProperties { private long bufferMemory = 33554432L; private boolean filterTransactionEntry = true; private int canalBatchSize = 50; + private Long canalGetTimeout; private List canalDestinations = new ArrayList(); @@ -151,6 +152,14 @@ public class KafkaProperties { this.canalBatchSize = canalBatchSize; } + public Long getCanalGetTimeout() { + return canalGetTimeout; + } + + public void setCanalGetTimeout(Long canalGetTimeout) { + this.canalGetTimeout = canalGetTimeout; + } + public List getCanalDestinations() { return canalDestinations; }