chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

HBase的etl初步完成

Browse Source
This commit is contained in:
mcy
2018-10-24 18:30:42 +08:00
parent 15e79c1400
commit 0a7251add0
17 changed files with 869 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
client-adapter/launcher/pom.xml
View File

@@ -50,6 +50,18 @@
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<!-- 单独引入rocketmq依赖 -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
</dependency>
<!-- 单独引入kafka依赖 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>1.1.1</version>
</dependency>
<!-- outer adapter jar with dependencies-->
<dependency>

6
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java
View File

@@ -4,6 +4,8 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@@ -15,6 +17,8 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
@ConfigurationProperties(prefix = "adapter.conf")
public class AdapterConfig {
private static Logger logger = LoggerFactory.getLogger(AdapterConfig.class);
private Map<String, DatasourceConfig> datasourceConfigs;
private List<String> adapterConfigs;
@@ -51,7 +55,7 @@ public class AdapterConfig {
try {
ds.init();
} catch (SQLException e) {
throw new RuntimeException(e);
logger.error("#Failed to initial datasource: " + datasourceConfig.getUrl(), e);
}
DatasourceConfig.DATA_SOURCES.put(entry.getKey(), ds);
}

92
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -7,6 +7,8 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.protocol.FlatMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,10 +26,10 @@ public abstract class AbstractCanalAdapterWorker {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected String canalDestination; // canal实例
protected List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
protected ExecutorService groupInnerExecutorService; // 组内工作线程池
protected volatile boolean running = false; // 是否运行中
protected String canalDestination; // canal实例
protected List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
protected ExecutorService groupInnerExecutorService; // 组内工作线程池
protected volatile boolean running = false; // 是否运行中
protected Thread thread = null;
protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
@@ -69,47 +71,47 @@ public abstract class AbstractCanalAdapterWorker {
}
}
// protected void writeOut(final FlatMessage flatMessage) {
// List<Future<Boolean>> futures = new ArrayList<>();
// // 组间适配器并行运行
// for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
// final List<CanalOuterAdapter> adapters = outerAdapters;
// futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
//
// @Override
// public Boolean call() {
// try {
// // 组内适配器穿行运行，尽量不要配置组内适配器
// for (CanalOuterAdapter c : adapters) {
// long begin = System.currentTimeMillis();
// Dml dml = MessageUtil.flatMessage2Dml(flatMessage);
// c.writeOut(dml);
// if (logger.isDebugEnabled()) {
// logger.debug("{} elapsed time: {}",
// c.getClass().getName(),
// (System.currentTimeMillis() - begin));
// }
// }
// return true;
// } catch (Exception e) {
// return false;
// }
// }
// }));
//
// // 等待所有适配器写入完成
// // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
// for (Future<Boolean> f : futures) {
// try {
// if (!f.get()) {
// logger.error("Outer adapter write failed");
// }
// } catch (InterruptedException | ExecutionException e) {
// // ignore
// }
// }
// }
// }
protected void writeOut(final FlatMessage flatMessage) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
final List<OuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
@Override
public Boolean call() {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
for (OuterAdapter c : adapters) {
long begin = System.currentTimeMillis();
Dml dml = MessageUtil.flatMessage2Dml(flatMessage);
c.sync(dml);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
c.getClass().getName(),
(System.currentTimeMillis() - begin));
}
}
return true;
} catch (Exception e) {
return false;
}
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
for (Future<Boolean> f : futures) {
try {
if (!f.get()) {
logger.error("Outer adapter write failed");
}
} catch (InterruptedException | ExecutionException e) {
// ignore
}
}
}
}
protected void writeOut(Message message, String topic) {
if (logger.isDebugEnabled()) {

148
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java Normal file
View File

@@ -0,0 +1,148 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
* kafka对应的client适配器工作线程
*
* @author machengyuan 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private KafkaCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
this.topic = topic;
this.canalDestination = topic;
this.flatMessage = flatMessage;
connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
// connector.setSessionTimeout(1L, TimeUnit.MINUTES);
// super.initSwitcher(topic);
}
@Override
public void start() {
if (!running) {
thread = new Thread(new Runnable() {
@Override
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}
}
@Override
public void stop() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
// if (switcher != null && !switcher.state()) {
// switcher.set(true);
// }
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("topic {} connectors' worker thread dead!", this.topic);
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) {
adapter.destroy();
}
}
logger.info("topic {} all connectors destroyed!", this.topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running)
;
ExecutorService executor = Executors.newSingleThreadExecutor();
// final AtomicBoolean executing = new AtomicBoolean(true);
while (running) {
try {
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
while (running) {
try {
// switcher.get(); //等待开关开启
List<?> messages;
if (!flatMessage) {
messages = connector.getWithoutAck();
} else {
messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null) {
for (final Object message : messages) {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
}
}
connector.ack();
} catch (CommitFailedException e) {
logger.warn(e.getMessage());
} catch (Exception e) {
logger.error(e.getMessage(), e);
TimeUnit.SECONDS.sleep(1L);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
executor.shutdown();
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

66
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -9,13 +9,14 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
/**
* 外部适配器的加载器
@@ -24,13 +25,15 @@ import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
*/
public class CanalAdapterLoader {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private CanalClientConfig canalClientConfig;
private CanalClientConfig canalClientConfig;
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
private ExtensionLoader<OuterAdapter> loader;
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){
this.canalClientConfig = canalClientConfig;
@@ -57,7 +60,7 @@ public class CanalAdapterLoader {
for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) {
List<OuterAdapter> canalOutConnectors = new ArrayList<>();
for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) {
for (OuterAdapterConfig c : connectorGroup.getOutAdapters()) {
loadConnector(c, canalOutConnectors);
}
canalOuterAdapterGroups.add(canalOutConnectors);
@@ -73,9 +76,43 @@ public class CanalAdapterLoader {
logger.info("Start adapter for canal instance: {} succeed", instance.getInstance());
}
}
// 初始化canal-client-mq的适配器
if (canalClientConfig.getMqTopics() != null) {
for (CanalClientConfig.MQTopic topic : canalClientConfig.getMqTopics()) {
for (CanalClientConfig.Group group : topic.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
for (OuterAdapterConfig config : group.getOutAdapters()) {
loadConnector(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig
.getBootstrapServers(), topic.getTopic(), group.getGroupId(), canalOuterAdapterGroups);
canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
} else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(
canalClientConfig.getBootstrapServers(),
topic.getTopic(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getFlatMessage());
canalMQWorker.put(topic.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start();
}
logger.info("Start adapter for canal-client rocketmq topic: {} succeed",
topic.getTopic() + "-" + group.getGroupId());
}
}
}
}
private void loadConnector(CanalOuterAdapterConfiguration config, List<OuterAdapter> canalOutConnectors) {
private void loadConnector(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
try {
OuterAdapter adapter = loader.getExtension(config.getName());
ClassLoader cl = Thread.currentThread().getContextClassLoader();
@@ -98,10 +135,19 @@ public class CanalAdapterLoader {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
for (CanalAdapterWorker v : canalWorkers.values()) {
final CanalAdapterWorker caw = v;
stopExecutorService.submit(() -> caw.stop());
stopExecutorService.submit(caw::stop);
}
stopExecutorService.shutdown();
}
if (canalMQWorker.size() > 0) {
ExecutorService stopMQWokerService = Executors.newFixedThreadPool(canalMQWorker.size());
for (AbstractCanalAdapterWorker tmp : canalMQWorker.values()) {
final AbstractCanalAdapterWorker worker = tmp;
stopMQWokerService.submit(worker::stop);
}
stopMQWokerService.shutdown();
}
logger.info("All canal adapters destroyed");
}
}

124
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java Normal file
View File

@@ -0,0 +1,124 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectorProvider;
import com.alibaba.otter.canal.protocol.Message;
/**
* kafka对应的client适配器工作线程
*
* @author machengyuan 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private RocketMQCanalConnector connector;
private String topic;
public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId,
List<List<OuterAdapter>> canalOuterAdapters){
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
this.topic = topic;
this.canalDestination = topic;
connector = RocketMQCanalConnectorProvider.newRocketMQConnector(nameServers, topic, groupId);
}
@Override
public void start() {
if (!running) {
thread = new Thread(new Runnable() {
@Override
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}
}
@Override
public void stop() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
logger.info("Stop topic {} out adapters begin", this.topic);
stopOutAdapters();
logger.info("Stop topic {} out adapters end", this.topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running)
;
ExecutorService executor = Executors.newSingleThreadExecutor();
while (running) {
try {
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
try {
// switcher.get(); //等待开关开启
final Message message = connector.getWithoutAck(1);
if (message != null) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
writeOut(message, topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
connector.ack(message.getId());
}
});
} else {
logger.debug("Message is null");
}
} catch (CommitFailedException e) {
logger.warn(e.getMessage());
} catch (Exception e) {
logger.error(e.getMessage(), e);
TimeUnit.SECONDS.sleep(1L);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
executor.shutdown();
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.stopRunning();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

8
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -59,13 +59,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
@Override
public void start() {
if (!running) {
thread = new Thread(new Runnable() {
@Override
public void run() {
process();
}
});
thread = new Thread(() -> process());
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;

28
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -1,20 +1,32 @@
server:
port: 8081
logging:
level:
com.alibaba.otter.canal.client.adapter.hbase: DEBUG
canal.conf:
canalServerHost: 127.0.0.1:11111
#zookeeperHosts: slave1:2181
#bootstrapServers: localhost:9092 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: false
# zookeeperHosts: slave1:2181
# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: true
canalInstances:
- instance: example
adapterGroups:
- outAdapters:
- name: logger
# - name: hbase
# hosts: slave1:2181
# properties: {znodeParent: "/hbase-unsecure"}
# - name: logger
- name: hbase
properties:
hbase.zookeeper.quorum: slave1
hbase.zookeeper.property.clientPort: 2181
zookeeper.znode.parent: /hbase-unsecure
# mqTopics:
# - mqMode: kafka
# topic: example
# groups:
# - groupId: g2
# outAdapters:
# - name: logger
adapter.conf:
datasourceConfigs: