chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

调整kafka同步发送和消费，确保消息不丢失

Browse Source
This commit is contained in:
mcy
2018-10-03 16:32:08 +08:00
parent 2e92ec3e4d
commit e4b6385dcc
3 changed files with 99 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java
View File

@@ -96,7 +96,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
while (!running)
;
ExecutorService executor = Executors.newSingleThreadExecutor();
final AtomicBoolean executing = new AtomicBoolean(true);
// final AtomicBoolean executing = new AtomicBoolean(true);
while (running) {
try {
logger.info("=============> Start to connect topic: {} <=============", this.topic);
@@ -116,36 +116,41 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
}
if (messages != null) {
for (final Object message : messages) {
executing.set(true);
if (message != null) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
executing.compareAndSet(true, false);
}
}
});
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
long currentTS = System.currentTimeMillis();
while (executing.get()) {
// 大于10秒未消费完ack一次keep alive
if (System.currentTimeMillis() - currentTS > 10000) {
connector.ack();
currentTS = System.currentTimeMillis();
}
}
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
// executing.set(true);
// if (message != null) {
// executor.submit(new Runnable() {
//
// @Override
// public void run() {
// try {
// if (message instanceof FlatMessage) {
// writeOut((FlatMessage) message);
// } else {
// writeOut((Message) message);
// }
// } catch (Exception e) {
// logger.error(e.getMessage(), e);
// } finally {
// executing.compareAndSet(true, false);
// }
// }
// });
//
// // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
// long currentTS = System.currentTimeMillis();
// while (executing.get()) {
// // 大于10秒未消费完ack一次keep alive
// if (System.currentTimeMillis() - currentTS > 10000) {
// connector.ack();
// currentTS = System.currentTimeMillis();
// }
// }
// }
}
}
connector.ack();