chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rdb etl count destination 接口适配

Browse Source 
es hbase 多源适配
This commit is contained in:
mcy
2018-11-12 15:07:46 +08:00
parent 7cca8857ff
commit 66912d3d05
18 changed files with 333 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java
View File

@@ -35,31 +35,32 @@ import com.alibaba.otter.canal.client.adapter.support.*;
@SPI("hbase")
public class HbaseAdapter implements OuterAdapter {
private static Logger logger = LoggerFactory.getLogger(HbaseAdapter.class);
private static Logger logger = LoggerFactory.getLogger(HbaseAdapter.class);
private static volatile Map<String, MappingConfig> hbaseMapping = null; // 文件名对应配置
private static volatile Map<String, MappingConfig> mappingConfigCache = null; // 库名-表名对应配置
private Map<String, MappingConfig> hbaseMapping = new HashMap<>(); // 文件名对应配置
private Map<String, MappingConfig> mappingConfigCache = new HashMap<>(); // 库名-表名对应配置
private Connection conn;
private HbaseSyncService hbaseSyncService;
private HbaseTemplate hbaseTemplate;
private Connection conn;
private HbaseSyncService hbaseSyncService;
private HbaseTemplate hbaseTemplate;
@Override
public void init(OuterAdapterConfig configuration) {
try {
if (mappingConfigCache == null) {
synchronized (MappingConfig.class) {
if (mappingConfigCache == null) {
hbaseMapping = MappingConfigLoader.load();
mappingConfigCache = new HashMap<>();
for (MappingConfig mappingConfig : hbaseMapping.values()) {
mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+ mappingConfig.getHbaseMapping().getDatabase() + "."
+ mappingConfig.getHbaseMapping().getTable(),
mappingConfig);
}
}
SPI spi = this.getClass().getAnnotation(SPI.class);
Map<String, MappingConfig> hbaseMappingTmp = MappingConfigLoader.load(spi.value());
// 过滤不匹配的key的配置
hbaseMappingTmp.forEach((key, mappingConfig) -> {
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|| mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey())) {
hbaseMapping.put(key, mappingConfig);
}
});
for (MappingConfig mappingConfig : hbaseMapping.values()) {
mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+ mappingConfig.getHbaseMapping().getDatabase() + "."
+ mappingConfig.getHbaseMapping().getTable(),
mappingConfig);
}
Map<String, String> properties = configuration.getProperties();

16
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -10,11 +10,13 @@ import java.util.*;
*/
public class MappingConfig {
private String dataSourceKey; // 数据源key
private String dataSourceKey; // 数据源key
private String destination; // canal实例或MQ的topic
private String outerAdapterKey; // adapter key
private HbaseMapping hbaseMapping; // hbase映射配置
private String destination; // canal实例或MQ的topic
private HbaseMapping hbaseMapping; // hbase映射配置
public String getDataSourceKey() {
return dataSourceKey;
@@ -24,6 +26,14 @@ public class MappingConfig {
this.dataSourceKey = dataSourceKey;
}
public String getOuterAdapterKey() {
return outerAdapterKey;
}
public void setOuterAdapterKey(String outerAdapterKey) {
this.outerAdapterKey = outerAdapterKey;
}
public String getDestination() {
return destination;
}

8
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java
View File

@@ -26,19 +26,17 @@ public class MappingConfigLoader {
private static Logger logger = LoggerFactory.getLogger(MappingConfigLoader.class);
private static final String BASE_PATH = "hbase";
/**
* 加载HBase表映射配置
*
* @return 配置名/配置文件名--对象
*/
public static Map<String, MappingConfig> load() {
public static Map<String, MappingConfig> load(String name) {
logger.info("## Start loading hbase mapping config ... ");
Map<String, MappingConfig> result = new LinkedHashMap<>();
Collection<String> configs = AdapterConfigs.get("hbase");
Collection<String> configs = AdapterConfigs.get(name);
if (configs == null) {
return result;
}
@@ -55,7 +53,7 @@ public class MappingConfigLoader {
String configContent = null;
if (c.endsWith(".yml")) {
configContent = readConfigContent(BASE_PATH + "/" + c);
configContent = readConfigContent(name + "/" + c);
}
// 简单配置database.table@datasourcekey?rowKey=key1,key2