chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1037 from rewerma/master

Browse Source 
完善adapter-launcher及hbase-adaper相关功能
This commit is contained in:
agapple
2018-10-25 20:19:48 +08:00
committed by GitHub
parent 346d091d00 ddfc2464f3
commit b5f5d3f0f0
48 changed files with 2167 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
client-adapter/launcher/assembly.xml Normal file
View File

@@ -0,0 +1,34 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>canal-adapter</id>
<formats>
<format>tar.gz</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}/lib</directory>
<outputDirectory>/lib/</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}/config</directory>
<outputDirectory>/config/</outputDirectory>
<includes>
<include>**</include>
</includes>
</fileSet>
</fileSets>
<baseDirectory>/</baseDirectory>
</assembly>

36
client-adapter/launcher/pom.xml
View File

@@ -50,6 +50,23 @@
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.10.0</version>
</dependency>
<!-- 单独引入rocketmq依赖 -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
</dependency>
<!-- 单独引入kafka依赖 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>1.1.1</version>
</dependency>
<!-- outer adapter jar with dependencies-->
<dependency>
@@ -81,6 +98,7 @@
</dependencies>
<build>
<finalName>canal-adapter-launcher</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
@@ -132,6 +150,24 @@
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>assembly.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

44
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java
View File

@@ -1,56 +1,12 @@
package com.alibaba.otter.canal.adapter.launcher;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import com.alibaba.otter.canal.adapter.launcher.loader.CanalAdapterLoader;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
@SpringBootApplication
public class CanalAdapterApplication {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterApplication.class);
private static CanalAdapterLoader adapterLoader;
public static void main(String[] args) {
new SpringApplicationBuilder(CanalAdapterApplication.class).run(args);
}
@Resource
private CanalClientConfig canalClientConf;
@PostConstruct
public void init() {
if (adapterLoader == null) {
try {
logger.info("## start the canal client adapters.");
adapterLoader = new CanalAdapterLoader(canalClientConf);
adapterLoader.init();
logger.info("## the canal client adapters are running now ......");
} catch (Throwable e) {
logger.error("## something goes wrong when starting up the canal client adapters:", e);
System.exit(0);
}
}
}
@PreDestroy
public void destroy() {
try {
logger.info("## stop the canal client adapters");
if (adapterLoader != null) {
adapterLoader.destroy();
}
} catch (Throwable e) {
logger.warn("## something goes wrong when stopping canal client adapters:", e);
} finally {
logger.info("## canal client adapters are down.");
}
}
}

114
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/EtlLock.java Normal file
View File

@@ -0,0 +1,114 @@
package com.alibaba.otter.canal.adapter.launcher.common;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.springframework.stereotype.Component;
import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient;
@Component
public class EtlLock {
private static final Map<String, ReentrantLock> LOCAL_LOCK = new ConcurrentHashMap<>();
private static final Map<String, InterProcessMutex> DISTRIBUTED_LOCK = new ConcurrentHashMap<>();
private static Mode mode = Mode.LOCAL;
@Resource
private CuratorClient curatorClient;
@PostConstruct
public void init() {
CuratorFramework curator = curatorClient.getCurator();
if (curator != null) {
mode = Mode.DISTRIBUTED;
} else {
mode = Mode.LOCAL;
}
}
private ReentrantLock getLock(String key) {
ReentrantLock lock = LOCAL_LOCK.get(key);
if (lock == null) {
synchronized (EtlLock.class) {
lock = LOCAL_LOCK.get(key);
if (lock == null) {
lock = new ReentrantLock();
LOCAL_LOCK.put(key, lock);
}
}
}
return lock;
}
private InterProcessMutex getRemoteLock(String key) {
InterProcessMutex lock = DISTRIBUTED_LOCK.get(key);
if (lock == null) {
synchronized (EtlLock.class) {
lock = DISTRIBUTED_LOCK.get(key);
if (lock == null) {
lock = new InterProcessMutex(curatorClient.getCurator(), key);
DISTRIBUTED_LOCK.put(key, lock);
}
}
}
return lock;
}
public void lock(String key) throws Exception {
if (mode == Mode.LOCAL) {
getLock(key).lock();
} else {
InterProcessMutex lock = getRemoteLock(key);
lock.acquire();
}
}
public boolean tryLock(String key, long timeout, TimeUnit unit) {
try {
if (mode == Mode.LOCAL) {
return getLock(key).tryLock(timeout, unit);
} else {
InterProcessMutex lock = getRemoteLock(key);
return lock.acquire(timeout, unit);
}
} catch (Exception e) {
return false;
}
}
public boolean tryLock(String key) {
try {
if (mode == Mode.LOCAL) {
return getLock(key).tryLock();
} else {
InterProcessMutex lock = getRemoteLock(key);
return lock.acquire(500, TimeUnit.MILLISECONDS);
}
} catch (Exception e) {
return false;
}
}
public void unlock(String key) {
if (mode == Mode.LOCAL) {
getLock(key).unlock();
} else {
InterProcessMutex lock = getRemoteLock(key);
try {
lock.release();
} catch (Exception e) {
// ignore
}
}
}
}

6
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java Normal file
View File

@@ -0,0 +1,6 @@
package com.alibaba.otter.canal.adapter.launcher.common;
public enum Mode {
LOCAL, // 本地模式
DISTRIBUTED // 分布式
}

208
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/SyncSwitch.java Normal file
View File

@@ -0,0 +1,208 @@
package com.alibaba.otter.canal.adapter.launcher.common;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.NodeCache;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.springframework.stereotype.Component;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
import com.alibaba.otter.canal.adapter.launcher.config.CuratorClient;
import com.alibaba.otter.canal.common.utils.BooleanMutex;
@Component
public class SyncSwitch {
private static final String SYN_SWITCH_ZK_NODE = "/sync-switch/";
private static final Map<String, BooleanMutex> LOCAL_LOCK = new ConcurrentHashMap<>();
private static final Map<String, BooleanMutex> DISTRIBUTED_LOCK = new ConcurrentHashMap<>();
private static Mode mode = Mode.LOCAL;
@Resource
private AdapterCanalConfig adapterCanalConfig;
@Resource
private CuratorClient curatorClient;
@PostConstruct
public void init() {
CuratorFramework curator = curatorClient.getCurator();
if (curator != null) {
mode = Mode.DISTRIBUTED;
DISTRIBUTED_LOCK.clear();
for (String destination : adapterCanalConfig.DESTINATIONS) {
// 对应每个destination注册锁
BooleanMutex mutex = new BooleanMutex(true);
initMutex(curator, destination, mutex);
DISTRIBUTED_LOCK.put(destination, mutex);
startListen(destination, mutex);
}
} else {
mode = Mode.LOCAL;
LOCAL_LOCK.clear();
for (String destination : adapterCanalConfig.DESTINATIONS) {
// 对应每个destination注册锁
LOCAL_LOCK.put(destination, new BooleanMutex(true));
}
}
}
private synchronized void startListen(String destination, BooleanMutex mutex) {
try {
String path = SYN_SWITCH_ZK_NODE + destination;
CuratorFramework curator = curatorClient.getCurator();
final NodeCache nodeCache = new NodeCache(curator, path);
nodeCache.start();
nodeCache.getListenable().addListener(() -> initMutex(curator, destination, mutex));
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
private synchronized void initMutex(CuratorFramework curator, String destination, BooleanMutex mutex) {
try {
String path = SYN_SWITCH_ZK_NODE + destination;
Stat stat = curator.checkExists().forPath(path);
if (stat == null) {
if (!mutex.state()) {
mutex.set(true);
}
} else {
String data = new String(curator.getData().forPath(path), StandardCharsets.UTF_8);
if ("on".equals(data)) {
if (!mutex.state()) {
mutex.set(true);
}
} else {
if (mutex.state()) {
mutex.set(false);
}
}
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
public synchronized void off(String destination) {
if (mode == Mode.LOCAL) {
BooleanMutex mutex = LOCAL_LOCK.get(destination);
if (mutex != null && mutex.state()) {
mutex.set(false);
}
} else {
try {
String path = SYN_SWITCH_ZK_NODE + destination;
try {
curatorClient.getCurator()
.create()
.creatingParentContainersIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(path, "off".getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
curatorClient.getCurator().setData().forPath(path, "off".getBytes(StandardCharsets.UTF_8));
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
public synchronized void on(String destination) {
if (mode == Mode.LOCAL) {
BooleanMutex mutex = LOCAL_LOCK.get(destination);
if (mutex != null && !mutex.state()) {
mutex.set(true);
}
} else {
try {
String path = SYN_SWITCH_ZK_NODE + destination;
try {
curatorClient.getCurator()
.create()
.creatingParentContainersIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(path, "on".getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
curatorClient.getCurator().setData().forPath(path, "on".getBytes(StandardCharsets.UTF_8));
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
public synchronized void release(String destination) {
if (mode == Mode.LOCAL) {
BooleanMutex mutex = LOCAL_LOCK.get(destination);
if (mutex != null && !mutex.state()) {
mutex.set(true);
}
}
if (mode == Mode.DISTRIBUTED) {
BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination);
if (mutex != null && !mutex.state()) {
mutex.set(true);
}
}
}
public Boolean status(String destination) {
if (mode == Mode.LOCAL) {
BooleanMutex mutex = LOCAL_LOCK.get(destination);
if (mutex != null) {
return mutex.state();
} else {
return null;
}
} else {
BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination);
if (mutex != null) {
return mutex.state();
} else {
return null;
}
}
}
public void get(String destination) throws InterruptedException {
if (mode == Mode.LOCAL) {
BooleanMutex mutex = LOCAL_LOCK.get(destination);
if (mutex != null) {
mutex.get();
}
} else {
BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination);
if (mutex != null) {
mutex.get();
}
}
}
public void get(String destination, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
if (mode == Mode.LOCAL) {
BooleanMutex mutex = LOCAL_LOCK.get(destination);
if (mutex != null) {
mutex.get(timeout, unit);
}
} else {
BooleanMutex mutex = DISTRIBUTED_LOCK.get(destination);
if (mutex != null) {
mutex.get(timeout, unit);
}
}
}
}

39
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java
View File

@@ -1,10 +1,45 @@
package com.alibaba.otter.canal.adapter.launcher.config;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@ConfigurationProperties(prefix = "canal.conf")
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
@Component
@ConfigurationProperties(prefix = "canal.conf")
public class AdapterCanalConfig extends CanalClientConfig {
public final Set<String> DESTINATIONS = new LinkedHashSet<>();
@Override
public void setCanalInstances(List<CanalInstance> canalInstances) {
super.setCanalInstances(canalInstances);
if (canalInstances != null) {
synchronized (DESTINATIONS) {
DESTINATIONS.clear();
for (CanalInstance canalInstance : canalInstances) {
DESTINATIONS.add(canalInstance.getInstance());
}
}
}
}
@Override
public void setMqTopics(List<MQTopic> mqTopics) {
super.setMqTopics(mqTopics);
if (mqTopics != null) {
synchronized (DESTINATIONS) {
DESTINATIONS.clear();
for (MQTopic mqTopic : mqTopics) {
DESTINATIONS.add(mqTopic.getTopic());
}
}
}
}
}

80
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfig.java Normal file
View File

@@ -0,0 +1,80 @@
package com.alibaba.otter.canal.adapter.launcher.config;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
@Component
@ConfigurationProperties(prefix = "adapter.conf")
public class AdapterConfig {
private static Logger logger = LoggerFactory.getLogger(AdapterConfig.class);
private Map<String, DatasourceConfig> datasourceConfigs;
private List<String> adapterConfigs;
public List<String> getAdapterConfigs() {
return adapterConfigs;
}
public Map<String, DatasourceConfig> getDatasourceConfigs() {
return datasourceConfigs;
}
public void setDatasourceConfigs(Map<String, DatasourceConfig> datasourceConfigs) {
this.datasourceConfigs = datasourceConfigs;
if (datasourceConfigs != null) {
for (Map.Entry<String, DatasourceConfig> entry : datasourceConfigs.entrySet()) {
DatasourceConfig datasourceConfig = entry.getValue();
// 加载数据源连接池
DruidDataSource ds = new DruidDataSource();
ds.setDriverClassName(datasourceConfig.getDriver());
ds.setUrl(datasourceConfig.getUrl());
ds.setUsername(datasourceConfig.getUsername());
ds.setPassword(datasourceConfig.getPassword());
ds.setInitialSize(1);
ds.setMinIdle(1);
ds.setMaxActive(datasourceConfig.getMaxActive());
ds.setMaxWait(60000);
ds.setTimeBetweenEvictionRunsMillis(60000);
ds.setMinEvictableIdleTimeMillis(300000);
ds.setPoolPreparedStatements(false);
ds.setMaxPoolPreparedStatementPerConnectionSize(20);
ds.setValidationQuery("select 1");
try {
ds.init();
} catch (SQLException e) {
logger.error("#Failed to initial datasource: " + datasourceConfig.getUrl(), e);
}
DatasourceConfig.DATA_SOURCES.put(entry.getKey(), ds);
}
}
}
public void setAdapterConfigs(List<String> adapterConfigs) {
this.adapterConfigs = adapterConfigs;
if (adapterConfigs != null) {
AdapterConfigs.clear();
for (String adapterConfig : adapterConfigs) {
int idx = adapterConfig.indexOf("/");
if (idx > -1) {
String type = adapterConfig.substring(0, idx);
String ymlFile = adapterConfig.substring(idx + 1);
AdapterConfigs.put(type, ymlFile);
}
}
}
}
}

35
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigPath.java
View File

@@ -1,35 +0,0 @@
package com.alibaba.otter.canal.adapter.launcher.config;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
@Component
@ConfigurationProperties(prefix = "adapter.conf.path")
public class AdapterConfigPath {
private List<String> adapterConfigs;
public List<String> getAdapterConfigs() {
return adapterConfigs;
}
public void setAdapterConfigs(List<String> adapterConfigs) {
this.adapterConfigs = adapterConfigs;
if (adapterConfigs != null) {
AdapterConfigs.configs.clear();
for (String adapterConfig : adapterConfigs) {
int idx = adapterConfig.indexOf("/");
if (idx > -1) {
String type = adapterConfig.substring(0, idx);
String ymlFile = adapterConfig.substring(idx + 1);
AdapterConfigs.configs.put(type, ymlFile);
}
}
}
}
}

36
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java Normal file
View File

@@ -0,0 +1,36 @@
package com.alibaba.otter.canal.adapter.launcher.config;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.springframework.stereotype.Component;
@Component
public class CuratorClient {
@Resource
private AdapterCanalConfig adapterCanalConfig;
private CuratorFramework curator = null;
@PostConstruct
public void init() {
if (adapterCanalConfig.getZookeeperHosts() != null) {
curator = CuratorFrameworkFactory.builder()
.connectString(adapterCanalConfig.getZookeeperHosts())
.retryPolicy(new ExponentialBackoffRetry(1000, 3))
.sessionTimeoutMs(6000)
.connectionTimeoutMs(3000)
.namespace("canal-adapter")
.build();
curator.start();
}
}
public CuratorFramework getCurator() {
return curator;
}
}

28
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java Normal file
View File

@@ -0,0 +1,28 @@
package com.alibaba.otter.canal.adapter.launcher.config;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class SpringContext implements ApplicationContextAware {
private static ApplicationContext context;
/*
* 注入ApplicationContext
*/
public void setApplicationContext(final ApplicationContext context) throws BeansException {
// 在加载Spring时自动获得context
SpringContext.context = context;
}
public static Object getBean(final String beanName) {
return SpringContext.context.getBean(beanName);
}
public static Object getBean(final Class<?> clz) {
return context.getBean(clz);
}
}

135
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -7,10 +7,14 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.protocol.FlatMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.protocol.Message;
@@ -24,34 +28,72 @@ public abstract class AbstractCanalAdapterWorker {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected String canalDestination; // canal实例
protected List<List<CanalOuterAdapter>> canalOuterAdapters; // 外部适配器
protected ExecutorService groupInnerExecutorService; // 组内工作线程池
protected volatile boolean running = false; // 是否运行中
protected String canalDestination; // canal实例
protected List<List<OuterAdapter>> canalOuterAdapters; // 外部适配器
protected ExecutorService groupInnerExecutorService; // 组内工作线程池
protected volatile boolean running = false; // 是否运行中
protected Thread thread = null;
protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
@Override
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
protected SyncSwitch syncSwitch;
public AbstractCanalAdapterWorker(){
syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
}
protected void writeOut(final Message message) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
final List<CanalOuterAdapter> adapters = outerAdapters;
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
final List<OuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(() -> {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
for (final OuterAdapter c : adapters) {
long begin = System.currentTimeMillis();
MessageUtil.parse4Dml(canalDestination, message, c::sync);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
c.getClass().getName(),
(System.currentTimeMillis() - begin));
}
}
return true;
} catch (Exception e) {
return false;
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
for (Future<Boolean> f : futures) {
try {
if (!f.get()) {
logger.error("Outer adapter write failed");
}
} catch (InterruptedException | ExecutionException e) {
// ignore
}
}
}
}
protected void writeOut(final FlatMessage flatMessage) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
final List<OuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
@Override
public Boolean call() {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
for (final CanalOuterAdapter c : adapters) {
for (OuterAdapter c : adapters) {
long begin = System.currentTimeMillis();
MessageUtil.parse4Dml(message, c::writeOut);
Dml dml = MessageUtil.flatMessage2Dml(canalDestination, flatMessage);
c.sync(dml);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
c.getClass().getName(),
@@ -79,63 +121,6 @@ public abstract class AbstractCanalAdapterWorker {
}
}
// protected void writeOut(final FlatMessage flatMessage) {
// List<Future<Boolean>> futures = new ArrayList<>();
// // 组间适配器并行运行
// for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
// final List<CanalOuterAdapter> adapters = outerAdapters;
// futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
//
// @Override
// public Boolean call() {
// try {
// // 组内适配器穿行运行，尽量不要配置组内适配器
// for (CanalOuterAdapter c : adapters) {
// long begin = System.currentTimeMillis();
// Dml dml = MessageUtil.flatMessage2Dml(flatMessage);
// c.writeOut(dml);
// if (logger.isDebugEnabled()) {
// logger.debug("{} elapsed time: {}",
// c.getClass().getName(),
// (System.currentTimeMillis() - begin));
// }
// }
// return true;
// } catch (Exception e) {
// return false;
// }
// }
// }));
//
// // 等待所有适配器写入完成
// // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
// for (Future<Boolean> f : futures) {
// try {
// if (!f.get()) {
// logger.error("Outer adapter write failed");
// }
// } catch (InterruptedException | ExecutionException e) {
// // ignore
// }
// }
// }
// }
protected void writeOut(Message message, String topic) {
if (logger.isDebugEnabled()) {
logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size());
}
long begin = System.currentTimeMillis();
writeOut(message);
long now = System.currentTimeMillis();
if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) {
logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin);
}
if (logger.isDebugEnabled()) {
logger.debug("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin);
}
}
protected void stopOutAdapters() {
if (thread != null) {
try {
@@ -146,8 +131,8 @@ public abstract class AbstractCanalAdapterWorker {
}
groupInnerExecutorService.shutdown();
logger.info("topic connectors' worker thread dead!");
for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
for (CanalOuterAdapter adapter : outerAdapters) {
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) {
adapter.destroy();
}
}

142
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java Normal file
View File

@@ -0,0 +1,142 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
* kafka对应的client适配器工作线程
*
* @author machengyuan 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private KafkaCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
this.topic = topic;
this.canalDestination = topic;
this.flatMessage = flatMessage;
connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
// connector.setSessionTimeout(1L, TimeUnit.MINUTES);
// super.initSwitcher(topic);
}
@Override
public void start() {
if (!running) {
thread = new Thread(() -> process());
thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}
}
@Override
public void stop() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
// if (switcher != null && !switcher.state()) {
// switcher.set(true);
// }
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
groupInnerExecutorService.shutdown();
logger.info("topic {} connectors' worker thread dead!", this.topic);
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) {
adapter.destroy();
}
}
logger.info("topic {} all connectors destroyed!", this.topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running)
;
ExecutorService executor = Executors.newSingleThreadExecutor();
// final AtomicBoolean executing = new AtomicBoolean(true);
while (running) {
try {
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
while (running) {
try {
// switcher.get(); //等待开关开启
List<?> messages;
if (!flatMessage) {
messages = connector.getWithoutAck();
} else {
messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null) {
for (final Object message : messages) {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
}
}
connector.ack();
} catch (CommitFailedException e) {
logger.warn(e.getMessage());
} catch (Exception e) {
logger.error(e.getMessage(), e);
TimeUnit.SECONDS.sleep(1L);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
executor.shutdown();
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

87
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -9,13 +9,14 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
/**
* 外部适配器的加载器
@@ -24,15 +25,17 @@ import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
*/
public class CanalAdapterLoader {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private CanalClientConfig canalClientConfig;
private CanalClientConfig canalClientConfig;
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
private ExtensionLoader<CanalOuterAdapter> loader;
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
public CanalAdapterLoader(CanalClientConfig canalClientConfig) {
private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){
this.canalClientConfig = canalClientConfig;
}
@@ -40,12 +43,7 @@ public class CanalAdapterLoader {
* 初始化canal-client
*/
public void init() {
// canal instances 和 mq topics 配置不能同时为空
// if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getMqTopics() == null) {
// throw new RuntimeException("Blank config property: canalInstances or canalMQTopics");
// }
loader = ExtensionLoader.getExtensionLoader(CanalOuterAdapter.class);
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
String canalServerHost = this.canalClientConfig.getCanalServerHost();
SocketAddress sa = null;
@@ -58,31 +56,67 @@ public class CanalAdapterLoader {
// 初始化canal-client的适配器
if (canalClientConfig.getCanalInstances() != null) {
for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) {
List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) {
List<CanalOuterAdapter> canalOutConnectors = new ArrayList<>();
for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) {
List<OuterAdapter> canalOutConnectors = new ArrayList<>();
for (OuterAdapterConfig c : connectorGroup.getOutAdapters()) {
loadConnector(c, canalOutConnectors);
}
canalOuterAdapterGroups.add(canalOutConnectors);
}
CanalAdapterWorker worker;
if (zkHosts != null) {
if (sa != null) {
worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups);
} else if (zkHosts != null) {
worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups);
} else {
worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups);
throw new RuntimeException("No canal server connector found");
}
canalWorkers.put(instance.getInstance(), worker);
worker.start();
logger.info("Start adapter for canal instance: {} succeed", instance.getInstance());
}
}
// 初始化canal-client-mq的适配器
if (canalClientConfig.getMqTopics() != null) {
for (CanalClientConfig.MQTopic topic : canalClientConfig.getMqTopics()) {
for (CanalClientConfig.Group group : topic.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
for (OuterAdapterConfig config : group.getOutAdapters()) {
loadConnector(config, canalOuterAdapters);
}
canalOuterAdapterGroups.add(canalOuterAdapters);
if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig
.getBootstrapServers(), topic.getTopic(), group.getGroupId(), canalOuterAdapterGroups);
canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
} else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(
canalClientConfig.getBootstrapServers(),
topic.getTopic(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getFlatMessage());
canalMQWorker.put(topic.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start();
}
logger.info("Start adapter for canal-client rocketmq topic: {} succeed",
topic.getTopic() + "-" + group.getGroupId());
}
}
}
}
private void loadConnector(CanalOuterAdapterConfiguration config, List<CanalOuterAdapter> canalOutConnectors) {
private void loadConnector(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
try {
CanalOuterAdapter adapter = loader.getExtension(config.getName());
OuterAdapter adapter = loader.getExtension(config.getName());
ClassLoader cl = Thread.currentThread().getContextClassLoader();
// 替换ClassLoader
Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
@@ -103,10 +137,19 @@ public class CanalAdapterLoader {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
for (CanalAdapterWorker v : canalWorkers.values()) {
final CanalAdapterWorker caw = v;
stopExecutorService.submit(() -> caw.stop());
stopExecutorService.submit(caw::stop);
}
stopExecutorService.shutdown();
}
if (canalMQWorker.size() > 0) {
ExecutorService stopMQWokerService = Executors.newFixedThreadPool(canalMQWorker.size());
for (AbstractCanalAdapterWorker tmp : canalMQWorker.values()) {
final AbstractCanalAdapterWorker worker = tmp;
stopMQWokerService.submit(worker::stop);
}
stopMQWokerService.shutdown();
}
logger.info("All canal adapters destroyed");
}
}

132
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java Normal file
View File

@@ -0,0 +1,132 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectorProvider;
import com.alibaba.otter.canal.protocol.Message;
/**
* kafka对应的client适配器工作线程
*
* @author machengyuan 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private RocketMQCanalConnector connector;
private String topic;
public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId,
List<List<OuterAdapter>> canalOuterAdapters){
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
this.topic = topic;
this.canalDestination = topic;
connector = RocketMQCanalConnectorProvider.newRocketMQConnector(nameServers, topic, groupId);
}
@Override
public void start() {
if (!running) {
thread = new Thread(new Runnable() {
@Override
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
running = true;
thread.start();
}
}
@Override
public void stop() {
try {
if (!running) {
return;
}
connector.stopRunning();
running = false;
logger.info("Stop topic {} out adapters begin", this.topic);
stopOutAdapters();
logger.info("Stop topic {} out adapters end", this.topic);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void process() {
while (!running)
;
ExecutorService executor = Executors.newSingleThreadExecutor();
while (running) {
try {
logger.info("=============> Start to connect topic: {} <=============", this.topic);
connector.connect();
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
try {
// switcher.get(); //等待开关开启
final Message message = connector.getWithoutAck(1);
if (message != null) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
if (logger.isDebugEnabled()) {
logger.debug("topic: {} batchId: {} batchSize: {} ", topic, message.getId(), message.getEntries().size());
}
long begin = System.currentTimeMillis();
writeOut(message);
long now = System.currentTimeMillis();
if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) {
logger.error("topic: {} batchId {} elapsed time: {} ms", topic, message.getId(), now - begin);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
connector.ack(message.getId());
}
});
} else {
logger.debug("Message is null");
}
} catch (CommitFailedException e) {
logger.warn(e.getMessage());
} catch (Exception e) {
logger.error(e.getMessage(), e);
TimeUnit.SECONDS.sleep(1L);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
executor.shutdown();
try {
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.stopRunning();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

71
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterService.java Normal file
View File

@@ -0,0 +1,71 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
@Component
public class CanalAdapterService {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterService.class);
private static CanalAdapterLoader adapterLoader;
@Resource
private AdapterCanalConfig adapterCanalConfig;
// 注入bean保证优先注册
@Resource
private AdapterConfig adapterConfig;
@Resource
private SpringContext springContext;
@Resource
private SyncSwitch syncSwitch;
@PostConstruct
public void init() {
if (adapterLoader == null) {
try {
logger.info("## start the canal client adapters.");
adapterLoader = new CanalAdapterLoader(adapterCanalConfig);
adapterLoader.init();
logger.info("## the canal client adapters are running now ......");
} catch (Throwable e) {
logger.error("## something goes wrong when starting up the canal client adapters:", e);
System.exit(0);
}
}
}
@PreDestroy
public void destroy() {
try {
logger.info("## stop the canal client adapters");
if (adapterLoader != null) {
adapterLoader.destroy();
}
for (DruidDataSource druidDataSource : DatasourceConfig.DATA_SOURCES.values()) {
try {
druidDataSource.close();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
} catch (Throwable e) {
logger.warn("## something goes wrong when stopping canal client adapters:", e);
} finally {
logger.info("## canal client adapters are down.");
}
}
}

48
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -3,10 +3,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.SocketAddress;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
@@ -31,7 +33,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(String canalDestination, SocketAddress address,
List<List<CanalOuterAdapter>> canalOuterAdapters){
List<List<OuterAdapter>> canalOuterAdapters){
this.canalOuterAdapters = canalOuterAdapters;
this.canalDestination = canalDestination;
groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
@@ -46,26 +48,18 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(String canalDestination, String zookeeperHosts,
List<List<CanalOuterAdapter>> canalOuterAdapters){
List<List<OuterAdapter>> canalOuterAdapters){
this.canalOuterAdapters = canalOuterAdapters;
this.canalDestination = canalDestination;
groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
// super.initSwitcher(canalDestination);
}
@Override
public void start() {
if (!running) {
thread = new Thread(new Runnable() {
@Override
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
@@ -79,13 +73,11 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
return;
}
// if (switcher != null && !switcher.state()) {
// switcher.set(true);
// }
connector.stopRunning();
running = false;
syncSwitch.release(canalDestination);
logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
if (thread != null) {
try {
@@ -96,8 +88,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
}
groupInnerExecutorService.shutdown();
logger.info("destination {} adapters' worker thread dead!", canalDestination);
for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
for (CanalOuterAdapter adapter : outerAdapters) {
for (List<OuterAdapter> outerAdapters : canalOuterAdapters) {
for (OuterAdapter adapter : outerAdapters) {
adapter.destroy();
}
}
@@ -112,22 +104,22 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
; // waiting until running == true
while (running) {
try {
// if (switcher != null) {
// switcher.get();
// }
syncSwitch.get(canalDestination);
logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
connector.connect();
logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination);
connector.subscribe();
logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
while (running) {
// try {
// if (switcher != null) {
// switcher.get();
// }
// } catch (TimeoutException e) {
// break;
// }
try {
syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
} catch (TimeoutException e) {
break;
}
if (!running) {
break;
}
// server配置canal.instance.network.soTimeout(默认: 30s)
// 范围内未与server交互，server将关闭本次socket连接

166
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java Normal file
View File

@@ -0,0 +1,166 @@
package com.alibaba.otter.canal.adapter.launcher.rest;
import java.util.*;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.*;
import com.alibaba.otter.canal.adapter.launcher.common.EtlLock;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterCanalConfig;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
import com.alibaba.otter.canal.client.adapter.support.Result;
@RestController
public class CommonRest {
private static Logger logger = LoggerFactory.getLogger(CommonRest.class);
private static final String ETL_LOCK_ZK_NODE = "/sync-etl/";
private ExtensionLoader<OuterAdapter> loader;
@Resource
private SyncSwitch syncSwitch;
@Resource
private EtlLock etlLock;
@Resource
private AdapterCanalConfig adapterCanalConfig;
@PostConstruct
public void init() {
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
}
/**
* ETL curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST
*
* @param type 类型 hbase, es
* @param task 任务名对应配置文件名 mytest_person2.yml
* @param params etl where条件参数, 为空全部导入
* @return
*/
@PostMapping("/etl/{type}/{task}")
public EtlResult etl(@PathVariable String type, @PathVariable String task,
@RequestParam(name = "params", required = false) String params) {
boolean locked = etlLock.tryLock(ETL_LOCK_ZK_NODE + type + "-" + task);
if (!locked) {
EtlResult result = new EtlResult();
result.setSucceeded(false);
result.setErrorMessage(task + " 有其他进程正在导入中, 请稍后再试");
return result;
}
try {
OuterAdapter adapter = loader.getExtension(type);
String destination = adapter.getDestination(task);
Boolean oriSwithcStatus = null;
if (destination != null) {
oriSwithcStatus = syncSwitch.status(destination);
syncSwitch.off(destination);
}
try {
List<String> paramArr = null;
if (params != null) {
String[] parmaArray = params.trim().split(";");
paramArr = Arrays.asList(parmaArray);
}
return adapter.etl(task, paramArr);
} finally {
if (destination != null && oriSwithcStatus != null && oriSwithcStatus) {
syncSwitch.on(destination);
}
}
} finally {
etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + task);
}
}
/**
* 统计总数 curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml
*
* @param type 类型 hbase, es
* @param task 任务名对应配置文件名 mytest_person2.yml
* @return
*/
@GetMapping("/count/{type}/{task}")
public Map<String, Object> count(@PathVariable String type, @PathVariable String task) {
OuterAdapter adapter = loader.getExtension(type);
return adapter.count(task);
}
/**
* 返回所有实例 curl http://127.0.0.1:8081/destinations
*/
@GetMapping("/destinations")
public List<Map<String, String>> destinations() {
List<Map<String, String>> result = new ArrayList<>();
Set<String> destinations = adapterCanalConfig.DESTINATIONS;
for (String destination : destinations) {
Map<String, String> resMap = new LinkedHashMap<>();
Boolean status = syncSwitch.status(destination);
String resStatus = "none";
if (status != null && status) {
resStatus = "on";
} else if (status != null && !status) {
resStatus = "off";
}
resMap.put("destination", destination);
resMap.put("status", resStatus);
result.add(resMap);
}
return result;
}
/**
* 实例同步开关 curl http://127.0.0.1:8081/syncSwitch/example/off -X PUT
*
* @param destination 实例名称
* @param status 开关状态: off on
* @return
*/
@PutMapping("/syncSwitch/{destination}/{status}")
public Result etl(@PathVariable String destination, @PathVariable String status) {
if (status.equals("on")) {
syncSwitch.on(destination);
logger.info("#Destination: {} sync on", destination);
return Result.createSuccess("实例: " + destination + " 开启同步成功");
} else if (status.equals("off")) {
syncSwitch.off(destination);
logger.info("#Destination: {} sync off", destination);
return Result.createSuccess("实例: " + destination + " 关闭同步成功");
} else {
Result result = new Result();
result.setCode(50000);
result.setMessage("实例: " + destination + " 操作失败");
return result;
}
}
/**
* 获取实例开关状态 curl http://127.0.0.1:8081/syncSwitch/example
*
* @param destination 实例名称
* @return
*/
@GetMapping("/syncSwitch/{destination}")
public Map<String, String> etl(@PathVariable String destination) {
Boolean status = syncSwitch.status(destination);
String resStatus = "none";
if (status != null && status) {
resStatus = "on";
} else if (status != null && !status) {
resStatus = "off";
}
Map<String, String> res = new LinkedHashMap<>();
res.put("stauts", resStatus);
return res;
}
}

44
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -1,21 +1,47 @@
server:
port: 8081
logging:
level:
com.alibaba.otter.canal.client.adapter.hbase: DEBUG
spring:
jackson:
date-format: yyyy-MM-dd HH:mm:ss
time-zone: GMT+8
default-property-inclusion: non_null
hbase.zookeeper.quorum: slave1
hbase.zookeeper.property.clientPort: 2181
hbase.zookeeper.znode.parent: /hbase-unsecure
canal.conf:
canalServerHost: 127.0.0.1:11111
#zookeeperHosts: slave1:2181
#bootstrapServers: localhost:9092 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: false
# zookeeperHosts: slave1:2181
# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: true
canalInstances:
- instance: example
adapterGroups:
- outAdapters:
- name: logger
# - name: hbase
# hosts: slave1:2181
# properties: {znodeParent: "/hbase-unsecure"}
# properties:
# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# mqTopics:
# - mqMode: kafka
# topic: example
# groups:
# - groupId: g2
# outAdapters:
# - name: logger
adapter.conf.path:
adapterConfigs:
- hbase/mytest_person2.yml
#adapter.conf:
# datasourceConfigs:
# defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
# username: root
# password: 121212
# adapterConfigs:
# - hbase/mytest_person2.yml