chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

规范注释

Browse Source
This commit is contained in:
mcy
2018-10-25 21:21:06 +08:00
parent ddfc2464f3
commit d70dc8b88c
29 changed files with 230 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -1,21 +1,18 @@
package com.alibaba.otter.canal.client.adapter.hbase.config;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
/**
* HBase表映射配置
*
* @author machengyuan 2018-8-21 下午06:45:49
* @author rewerma 2018-8-21 下午06:45:49
* @version 1.0.0
*/
public class MappingConfig {
private String dataSourceKey;
private HbaseOrm hbaseOrm;
private String dataSourceKey; // 数据源key
private HbaseOrm hbaseOrm; // hbase映射配置
public String getDataSourceKey() {
return dataSourceKey;
@@ -130,7 +127,7 @@ public class MappingConfig {
}
public enum Mode {
STRING("STRING"), NATIVE("NATIVE"), PHOENIX("PHOENIX");
STRING("STRING"), NATIVE("NATIVE"), PHOENIX("PHOENIX");
private String type;
@@ -145,23 +142,23 @@ public class MappingConfig {
public static class HbaseOrm {
private Mode mode = Mode.STRING;
private String destination;
private String database;
private String table;
private String hbaseTable;
private String family = "CF";
private boolean uppercaseQualifier = true;
private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表
private String rowKey; // 指定复合主键为rowKey
private Map<String, String> columns;
private ColumnItem rowKeyColumn;
private String etlCondition;
private Mode mode = Mode.STRING; // hbase默认转换格式
private String destination; // canal实例或MQ的topic
private String database; // 数据库名或schema名
private String table; // 表面名
private String hbaseTable; // hbase表名
private String family = "CF"; // 默认统一column family
private boolean uppercaseQualifier = true; // 是否转大写
private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表
private String rowKey; // 指定复合主键为rowKey
private Map<String, String> columns; // 字段映射
private ColumnItem rowKeyColumn; // rowKey字段
private String etlCondition; // etl条件sql
private Map<String, ColumnItem> columnItems = new LinkedHashMap<>();
private Set<String> families = new LinkedHashSet<>();
private Map<String, ColumnItem> columnItems = new LinkedHashMap<>(); // 转换后的字段映射列表
private Set<String> families = new LinkedHashSet<>(); // column family列表
private int readBatch = 5000;
private int commitBatch = 5000;
private int commitBatch = 5000; // etl等批量提交大小
public Mode getMode() {
return mode;

2
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java
View File

@@ -18,7 +18,7 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
/**
* HBase表映射配置加载器
*
* @author machengyuan 2018-8-21 下午06:45:49
* @author rewerma 2018-8-21 下午06:45:49
* @version 1.0.0
*/
public class MappingConfigLoader {

32
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -24,6 +24,12 @@ import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.google.common.base.Joiner;
/**
* HBase ETL 操作业务类
*
* @author rewerma @ 2018-10-20
* @version 1.0.0
*/
public class HbaseEtlService {
private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class);
@@ -66,6 +72,12 @@ public class HbaseEtlService {
}
}
/**
* 建表
*
* @param hbaseTemplate
* @param config
*/
public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) {
try {
// 判断hbase表是否存在，不存在则建表
@@ -79,6 +91,15 @@ public class HbaseEtlService {
}
}
/**
* 导入数据
*
* @param ds 数据源
* @param hbaseTemplate hbaseTemplate
* @param config 配置
* @param params 筛选条件
* @return 导入结果
*/
public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config,
List<String> params) {
EtlResult etlResult = new EtlResult();
@@ -209,6 +230,17 @@ public class HbaseEtlService {
return etlResult;
}
/**
* 执行导入
*
* @param ds
* @param sql
* @param hbaseOrm
* @param hbaseTemplate
* @param successCount
* @param errMsg
* @return
*/
private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm,
HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) {
try {

2
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -13,7 +13,7 @@ import com.alibaba.otter.canal.client.adapter.support.Dml;
/**
* HBase同步操作业务
*
* @author machengyuan 2018-8-21 下午06:45:49
* @author rewerma 2018-8-21 下午06:45:49
* @version 1.0.0
*/
public class HbaseSyncService {