chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rdb etl count destination 接口适配

Browse Source 
es hbase 多源适配
This commit is contained in:
mcy
2018-11-12 15:07:46 +08:00
parent 7cca8857ff
commit 66912d3d05
18 changed files with 333 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -33,6 +33,8 @@ public class CanalAdapterLoader {
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private Map<String, OuterAdapter> outerAdapters = new HashMap<>(); // 配置文件对应adapter
private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){
@@ -53,8 +55,8 @@ public class CanalAdapterLoader {
}
String zkHosts = this.canalClientConfig.getZookeeperHosts();
// 初始化canal-client的适配器
if (canalClientConfig.getCanalInstances() != null) {
// 初始化canal-client的适配器
for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
@@ -83,10 +85,8 @@ public class CanalAdapterLoader {
worker.start();
logger.info("Start adapter for canal instance: {} succeed", instance.getInstance());
}
}
// 初始化canal-client-mq的适配器
if (canalClientConfig.getMqTopics() != null) {
} else if (canalClientConfig.getMqTopics() != null) {
// 初始化canal-client-mq的适配器
for (CanalClientConfig.MQTopic topic : canalClientConfig.getMqTopics()) {
for (CanalClientConfig.MQGroup group : topic.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
@@ -124,11 +124,11 @@ public class CanalAdapterLoader {
private void loadConnector(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
try {
OuterAdapter adapter;
if ("rdb".equalsIgnoreCase(config.getName())) {
adapter = loader.getExtension(config.getName(), config.getKey());
} else {
adapter = loader.getExtension(config.getName());
}
// if ("rdb".equalsIgnoreCase(config.getName())) {
adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
// } else {
// adapter = loader.getExtension(config.getName());
// }
ClassLoader cl = Thread.currentThread().getContextClassLoader();
// 替换ClassLoader
Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());

51
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java
View File

@@ -46,17 +46,17 @@ public class CommonRest {
}
/**
* ETL curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST
*
* ETL curl http://127.0.0.1:8081/etl/rdb/oracle1/mytest_user.yml -X POST
*
* @param type 类型 hbase, es
* @param task 任务名对应配置文件名 mytest_person2.yml
* @param key adapter key
* @param task 任务名对应配置文件名 mytest_user.yml
* @param params etl where条件参数, 为空全部导入
* @return
*/
@PostMapping("/etl/{type}/{task}")
public EtlResult etl(@PathVariable String type, @PathVariable String task,
@PostMapping("/etl/{type}/{key}/{task}")
public EtlResult etl(@PathVariable String type, @PathVariable String key, @PathVariable String task,
@RequestParam(name = "params", required = false) String params) {
OuterAdapter adapter = loader.getExtension(type);
OuterAdapter adapter = loader.getExtension(type, key);
String destination = adapter.getDestination(task);
String lockKey = destination == null ? task : destination;
@@ -83,12 +83,11 @@ public class CommonRest {
}
}
try {
List<String> paramArr = null;
List<String> paramArray = null;
if (params != null) {
String[] parmaArray = params.trim().split(";");
paramArr = Arrays.asList(parmaArray);
paramArray = Arrays.asList(params.trim().split(";"));
}
return adapter.etl(task, paramArr);
return adapter.etl(task, paramArray);
} finally {
if (destination != null && oriSwitchStatus != null && oriSwitchStatus) {
syncSwitch.on(destination);
@@ -101,6 +100,33 @@ public class CommonRest {
}
}
/**
* ETL curl http://127.0.0.1:8081/etl/hbase/mytest_person2.yml -X POST
*
* @param type 类型 hbase, es
* @param task 任务名对应配置文件名 mytest_person2.yml
* @param params etl where条件参数, 为空全部导入
*/
@PostMapping("/etl/{type}/{task}")
public EtlResult etl(@PathVariable String type, @PathVariable String task,
@RequestParam(name = "params", required = false) String params) {
return etl(type, null, task, params);
}
/**
* 统计总数 curl http://127.0.0.1:8081/count/rdb/oracle1/mytest_user.yml
*
* @param type 类型 hbase, es
* @param key adapter key
* @param task 任务名对应配置文件名 mytest_person2.yml
* @return
*/
@GetMapping("/count/{type}/{key}/{task}")
public Map<String, Object> count(@PathVariable String type, @PathVariable String key, @PathVariable String task) {
OuterAdapter adapter = loader.getExtension(type, key);
return adapter.count(task);
}
/**
* 统计总数 curl http://127.0.0.1:8081/count/hbase/mytest_person2.yml
*
@@ -110,8 +136,7 @@ public class CommonRest {
*/
@GetMapping("/count/{type}/{task}")
public Map<String, Object> count(@PathVariable String type, @PathVariable String task) {
OuterAdapter adapter = loader.getExtension(type);
return adapter.count(task);
return count(type, null, task);
}
/**