From b5e05c8c77fd7091901500360e28f568b20ca89b Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 28 Aug 2018 13:51:23 +0800 Subject: [PATCH 1/2] =?UTF-8?q?kafka=20client=20keep=20alive=E9=97=B4?= =?UTF-8?q?=E9=9A=94=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/adapter/loader/CanalAdapterKafkaWorker.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java b/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java index 5bccb51a..aecaceb2 100644 --- a/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java +++ b/client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java @@ -143,9 +143,13 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { }); // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端 + long currentTS = System.currentTimeMillis(); while (executing.get()) { - connector.ack(); - Thread.sleep(500); + // 大于1分钟未消费完ack一次keep alive + if (System.currentTimeMillis() - currentTS > 60000) { + connector.ack(); + currentTS = System.currentTimeMillis(); + } } } else { connector.ack(); From bcf30a9a05934e30220b9f8e6aec41f2d0246b6f Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 28 Aug 2018 15:52:36 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=8E=BB=E6=8E=89kafka=20canalBatchSize?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E9=BB=98=E8=AE=A450?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deployer/src/main/resources/kafka.yml | 2 -- .../java/com/alibaba/otter/canal/kafka/KafkaProperties.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/deployer/src/main/resources/kafka.yml b/deployer/src/main/resources/kafka.yml index 93d4d557..8abf4934 100644 --- a/deployer/src/main/resources/kafka.yml +++ b/deployer/src/main/resources/kafka.yml @@ -3,8 +3,6 @@ retries: 0 batchSize: 16384 lingerMs: 1 bufferMemory: 33554432 -# canal的批次大小,单位 k,量大建议改为1M -canalBatchSize: 50 filterTransactionEntry: true canalDestinations: diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java b/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java index 140ce965..9194c0ab 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/KafkaProperties.java @@ -19,7 +19,7 @@ public class KafkaProperties { private int lingerMs = 1; private long bufferMemory = 33554432L; private boolean filterTransactionEntry = true; - private int canalBatchSize = 5; + private int canalBatchSize = 50; private List canalDestinations = new ArrayList();