chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

代码整理

Browse Source
This commit is contained in:
mcy
2018-10-26 10:33:16 +08:00
parent d70dc8b88c
commit 848cbe11dd
5 changed files with 106 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,20 +2,19 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.protocol.FlatMessage;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil; import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
/** /**
@@ -44,21 +43,21 @@ public abstract class AbstractCanalAdapterWorker {
protected void writeOut(final Message message) { protected void writeOut(final Message message) {
List<Future<Boolean>> futures = new ArrayList<>(); List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行 // 组间适配器并行运行
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) { canalOuterAdapters.forEach(outerAdapters -> {
final List<OuterAdapter> adapters = outerAdapters; final List<OuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(() -> { futures.add(groupInnerExecutorService.submit(() -> {
try { try {
// 组内适配器穿行运行，尽量不要配置组内适配器 // 组内适配器穿行运行，尽量不要配置组内适配器
for (final OuterAdapter c : adapters) { adapters.forEach(adapter -> {
long begin = System.currentTimeMillis(); long begin = System.currentTimeMillis();
MessageUtil.parse4Dml(canalDestination, message, c::sync); MessageUtil.parse4Dml(canalDestination, message, adapter::sync);
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}", logger.debug("{} elapsed time: {}",
c.getClass().getName(), adapter.getClass().getName(),
(System.currentTimeMillis() - begin)); (System.currentTimeMillis() - begin));
} }
} });
return true; return true;
} catch (Exception e) { } catch (Exception e) {
return false; return false;
@@ -67,79 +66,94 @@ public abstract class AbstractCanalAdapterWorker {
// 等待所有适配器写入完成 // 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
for (Future<Boolean> f : futures) { futures.forEach(future -> {
try { try {
if (!f.get()) { if (!future.get()) {
logger.error("Outer adapter write failed"); logger.error("Outer adapter write failed");
} }
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
// ignore // ignore
} }
} });
} });
} }
protected void writeOut(final FlatMessage flatMessage) { protected void writeOut(final FlatMessage flatMessage) {
List<Future<Boolean>> futures = new ArrayList<>(); List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行 // 组间适配器并行运行
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) { canalOuterAdapters.forEach(outerAdapters -> {
final List<OuterAdapter> adapters = outerAdapters; futures.add(groupInnerExecutorService.submit(() -> {
futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() { try {
// 组内适配器穿行运行，尽量不要配置组内适配器
@Override outerAdapters.forEach(adapter -> {
public Boolean call() { long begin = System.currentTimeMillis();
try { Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage);
// 组内适配器穿行运行，尽量不要配置组内适配器 adapter.sync(dml);
for (OuterAdapter c : adapters) { if (logger.isDebugEnabled()) {
long begin = System.currentTimeMillis(); logger.debug("{} elapsed time: {}",
Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage); adapter.getClass().getName(),
c.sync(dml); (System.currentTimeMillis() - begin));
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
c.getClass().getName(),
(System.currentTimeMillis() - begin));
}
} }
return true; });
} catch (Exception e) { return true;
return false; } catch (Exception e) {
} return false;
} }
})); }));
// 等待所有适配器写入完成 // 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成 // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
for (Future<Boolean> f : futures) { futures.forEach(future -> {
try { try {
if (!f.get()) { if (!future.get()) {
logger.error("Outer adapter write failed"); logger.error("Outer adapter write failed");
} }
} catch (InterruptedException | ExecutionException e) { } catch (InterruptedException | ExecutionException e) {
// ignore // ignore
} }
} });
});
}
public void start() {
if (!running) {
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
} }
} }
protected void stopOutAdapters() { protected abstract void process();
if (thread != null) {
try { public void stop() {
thread.join(); try {
} catch (InterruptedException e) { if (!running) {
// ignore return;
} }
}
groupInnerExecutorService.shutdown(); closeConnection();
logger.info("topic connectors' worker thread dead!"); running = false;
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) { syncSwitch.release(canalDestination);
adapter.destroy();
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
} }
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters worker thread dead!", canalDestination);
canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
} }
logger.info("topic all connectors destroyed!");
} }
public abstract void start(); protected abstract void closeConnection();
public abstract void stop();
} }

49
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -37,59 +37,18 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
this.flatMessage = flatMessage; this.flatMessage = flatMessage;
connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage); connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
// connector.setSessionTimeout(1L, TimeUnit.MINUTES); // connector.setSessionTimeout(1L, TimeUnit.MINUTES);
// super.initSwitcher(topic);
} }
@Override @Override
public void start() { protected void closeConnection(){
if (!running) { connector.stopRunning();
thread = new Thread(() -> process());
thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}
} }
@Override @Override
public void stop() { protected void process() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
// if (switcher != null && !switcher.state()) {
// switcher.set(true);
// }
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("topic {} connectors' worker thread dead!", this.topic);
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) {
adapter.destroy();
}
}
logger.info("topic {} all connectors destroyed!", this.topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running) while (!running)
; ;
ExecutorService executor = Executors.newSingleThreadExecutor(); ExecutorService executor = Executors.newSingleThreadExecutor();
// final AtomicBoolean executing = new AtomicBoolean(true);
while (running) { while (running) {
try { try {
logger.info("=============> Start to connect topic: {} <=============", this.topic); logger.info("=============> Start to connect topic: {} <=============", this.topic);
@@ -99,7 +58,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic); logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
while (running) { while (running) {
try { try {
// switcher.get(); //等待开关开启 syncSwitch.get(canalDestination);
List<?> messages; List<?> messages;
if (!flatMessage) { if (!flatMessage) {

14
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -135,20 +135,18 @@ public class CanalAdapterLoader {
public void destroy() { public void destroy() {
if (canalWorkers.size() > 0) { if (canalWorkers.size() > 0) {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
for (CanalAdapterWorker v : canalWorkers.values()) { for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
final CanalAdapterWorker caw = v; stopExecutorService.submit(canalAdapterWorker::stop);
stopExecutorService.submit(caw::stop);
} }
stopExecutorService.shutdown(); stopExecutorService.shutdown();
} }
if (canalMQWorker.size() > 0) { if (canalMQWorker.size() > 0) {
ExecutorService stopMQWokerService = Executors.newFixedThreadPool(canalMQWorker.size()); ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
for (AbstractCanalAdapterWorker tmp : canalMQWorker.values()) { for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
final AbstractCanalAdapterWorker worker = tmp; stopMQWorkerService.submit(canalAdapterMQWorker::stop);
stopMQWokerService.submit(worker::stop);
} }
stopMQWokerService.shutdown(); stopMQWorkerService.shutdown();
} }
logger.info("All canal adapters destroyed"); logger.info("All canal adapters destroyed");
} }

68
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -36,38 +36,12 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
} }
@Override @Override
public void start() { protected void closeConnection() {
if (!running) { connector.stopRunning();
thread = new Thread(new Runnable() {
@Override
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}
} }
@Override @Override
public void stop() { protected void process() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
logger.info("Stop topic {} out adapters begin", this.topic);
stopOutAdapters();
logger.info("Stop topic {} out adapters end", this.topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running) while (!running)
; ;
ExecutorService executor = Executors.newSingleThreadExecutor(); ExecutorService executor = Executors.newSingleThreadExecutor();
@@ -84,25 +58,27 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
final Message message = connector.getWithoutAck(1); final Message message = connector.getWithoutAck(1);
if (message != null) { if (message != null) {
executor.submit(new Runnable() { executor.submit(() -> {
try {
@Override if (logger.isDebugEnabled()) {
public void run() { logger.debug("topic: {} batchId: {} batchSize: {} ",
try { topic,
if (logger.isDebugEnabled()) { message.getId(),
logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size()); message.getEntries().size());
}
long begin = System.currentTimeMillis();
writeOut(message);
long now = System.currentTimeMillis();
if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) {
logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} }
connector.ack(message.getId()); long begin = System.currentTimeMillis();
writeOut(message);
long now = System.currentTimeMillis();
if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) {
logger.error("topic: {} batchId {} elapsed time: {} ms",
topic,
message.getId(),
now - begin);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} }
connector.ack(message.getId());
}); });
} else { } else {
logger.debug("Message is null"); logger.debug("Message is null");

55
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -57,49 +57,12 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
} }
@Override @Override
public void start() { protected void closeConnection() {
if (!running) { connector.stopRunning();
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
} }
@Override @Override
public void stop() { protected void process() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters' worker thread dead!", canalDestination);
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) {
adapter.destroy();
}
}
logger.info("destination {} all adapters destroyed!", canalDestination);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running) while (!running)
; // waiting until running == true ; // waiting until running == true
while (running) { while (running) {
@@ -129,11 +92,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
int size = message.getEntries().size(); int size = message.getEntries().size();
if (batchId == -1 || size == 0) { if (batchId == -1 || size == 0) {
try { Thread.sleep(1000);
Thread.sleep(1000);
} catch (InterruptedException e) {
// ignore
}
} else { } else {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} batchSize: {} ", logger.debug("destination: {} batchId: {} batchSize: {} ",
@@ -143,18 +102,18 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
} }
long begin = System.currentTimeMillis(); long begin = System.currentTimeMillis();
writeOut(message); writeOut(message);
long now = System.currentTimeMillis();
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} elapsed time: {} ms", logger.debug("destination: {} batchId: {} elapsed time: {} ms",
this.canalDestination, this.canalDestination,
batchId, batchId,
now - begin); System.currentTimeMillis() - begin);
} }
} }
connector.ack(batchId); // 提交确认 connector.ack(batchId); // 提交确认
} catch (Exception e) { } catch (Exception e) {
connector.rollback(batchId); // 处理失败, 回滚数据 connector.rollback(batchId); // 处理失败, 回滚数据
throw e; logger.error("sync error!", e);
Thread.sleep(500);
} }
} }