chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

修改hbase配置项名

Browse Source
This commit is contained in:
mcy
2018-10-27 20:03:26 +08:00
parent 0d22dbc982
commit 87ac3daecc
7 changed files with 148 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java
View File

@@ -53,9 +53,9 @@ public class HbaseAdapter implements OuterAdapter {
hbaseMapping = MappingConfigLoader.load(); hbaseMapping = MappingConfigLoader.load();
mappingConfigCache = new HashMap<>(); mappingConfigCache = new HashMap<>();
for (MappingConfig mappingConfig : hbaseMapping.values()) { for (MappingConfig mappingConfig : hbaseMapping.values()) {
mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseOrm().getDestination()) mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getHbaseMapping().getDestination())
+ "." + mappingConfig.getHbaseOrm().getDatabase() + "." + "." + mappingConfig.getHbaseMapping().getDatabase() + "."
+ mappingConfig.getHbaseOrm().getTable(), + mappingConfig.getHbaseMapping().getTable(),
mappingConfig); mappingConfig);
} }
} }
@@ -136,7 +136,7 @@ public class HbaseAdapter implements OuterAdapter {
@Override @Override
public Map<String, Object> count(String task) { public Map<String, Object> count(String task) {
MappingConfig config = hbaseMapping.get(task); MappingConfig config = hbaseMapping.get(task);
String hbaseTable = config.getHbaseOrm().getHbaseTable(); String hbaseTable = config.getHbaseMapping().getHbaseTable();
long rowCount = 0L; long rowCount = 0L;
try { try {
HTable table = (HTable) conn.getTable(TableName.valueOf(hbaseTable)); HTable table = (HTable) conn.getTable(TableName.valueOf(hbaseTable));
@@ -169,8 +169,8 @@ public class HbaseAdapter implements OuterAdapter {
@Override @Override
public String getDestination(String task) { public String getDestination(String task) {
MappingConfig config = hbaseMapping.get(task); MappingConfig config = hbaseMapping.get(task);
if (config != null && config.getHbaseOrm() != null) { if (config != null && config.getHbaseMapping() != null) {
return config.getHbaseOrm().getDestination(); return config.getHbaseMapping().getDestination();
} }
return null; return null;
} }

42
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -10,9 +10,9 @@ import java.util.*;
*/ */
public class MappingConfig { public class MappingConfig {
private String dataSourceKey; // 数据源key private String dataSourceKey; // 数据源key
private HbaseOrm hbaseOrm; // hbase映射配置 private HbaseMapping hbaseMapping; // hbase映射配置
public String getDataSourceKey() { public String getDataSourceKey() {
return dataSourceKey; return dataSourceKey;
@@ -22,28 +22,28 @@ public class MappingConfig {
this.dataSourceKey = dataSourceKey; this.dataSourceKey = dataSourceKey;
} }
public HbaseOrm getHbaseOrm() { public HbaseMapping getHbaseMapping() {
return hbaseOrm; return hbaseMapping;
} }
public void setHbaseOrm(HbaseOrm hbaseOrm) { public void setHbaseMapping(HbaseMapping hbaseMapping) {
this.hbaseOrm = hbaseOrm; this.hbaseMapping = hbaseMapping;
} }
public void validate() { public void validate() {
if (hbaseOrm.database == null || hbaseOrm.database.isEmpty()) { if (hbaseMapping.database == null || hbaseMapping.database.isEmpty()) {
throw new NullPointerException("hbaseOrm.database"); throw new NullPointerException("hbaseMapping.database");
} }
if (hbaseOrm.table == null || hbaseOrm.table.isEmpty()) { if (hbaseMapping.table == null || hbaseMapping.table.isEmpty()) {
throw new NullPointerException("hbaseOrm.table"); throw new NullPointerException("hbaseMapping.table");
} }
if (hbaseOrm.hbaseTable == null || hbaseOrm.hbaseTable.isEmpty()) { if (hbaseMapping.hbaseTable == null || hbaseMapping.hbaseTable.isEmpty()) {
throw new NullPointerException("hbaseOrm.hbaseTable"); throw new NullPointerException("hbaseMapping.hbaseTable");
} }
if (hbaseOrm.mode == null) { if (hbaseMapping.mode == null) {
throw new NullPointerException("hbaseOrm.mode"); throw new NullPointerException("hbaseMapping.mode");
} }
if (hbaseOrm.rowKey != null && hbaseOrm.rowKeyColumn != null) { if (hbaseMapping.rowKey != null && hbaseMapping.rowKeyColumn != null) {
throw new RuntimeException("已配置了复合主键作为RowKey，无需再指定RowKey列"); throw new RuntimeException("已配置了复合主键作为RowKey，无需再指定RowKey列");
} }
} }
@@ -55,12 +55,12 @@ public class MappingConfig {
MappingConfig config = (MappingConfig) o; MappingConfig config = (MappingConfig) o;
return hbaseOrm != null ? hbaseOrm.equals(config.hbaseOrm) : config.hbaseOrm == null; return hbaseMapping != null ? hbaseMapping.equals(config.hbaseMapping) : config.hbaseMapping == null;
} }
@Override @Override
public int hashCode() { public int hashCode() {
return hbaseOrm != null ? hbaseOrm.hashCode() : 0; return hbaseMapping != null ? hbaseMapping.hashCode() : 0;
} }
public static class ColumnItem { public static class ColumnItem {
@@ -140,7 +140,7 @@ public class MappingConfig {
} }
} }
public static class HbaseOrm { public static class HbaseMapping {
private Mode mode = Mode.STRING; // hbase默认转换格式 private Mode mode = Mode.STRING; // hbase默认转换格式
private String destination; // canal实例或MQ的topic private String destination; // canal实例或MQ的topic
@@ -349,10 +349,10 @@ public class MappingConfig {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
HbaseOrm hbaseOrm = (HbaseOrm) o; HbaseMapping hbaseMapping = (HbaseMapping) o;
if (table != null ? !table.equals(hbaseOrm.table) : hbaseOrm.table != null) return false; if (table != null ? !table.equals(hbaseMapping.table) : hbaseMapping.table != null) return false;
return hbaseTable != null ? hbaseTable.equals(hbaseOrm.hbaseTable) : hbaseOrm.hbaseTable == null; return hbaseTable != null ? hbaseTable.equals(hbaseMapping.hbaseTable) : hbaseMapping.hbaseTable == null;
} }
@Override @Override

18
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java
View File

@@ -80,21 +80,21 @@ public class MappingConfigLoader {
if (dbTable.length == 2) { if (dbTable.length == 2) {
config = new MappingConfig(); config = new MappingConfig();
MappingConfig.HbaseOrm hbaseOrm = new MappingConfig.HbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = new MappingConfig.HbaseMapping();
hbaseOrm.setHbaseTable(dbTable[0].toUpperCase() + "." + dbTable[1].toUpperCase()); hbaseMapping.setHbaseTable(dbTable[0].toUpperCase() + "." + dbTable[1].toUpperCase());
hbaseOrm.setAutoCreateTable(true); hbaseMapping.setAutoCreateTable(true);
hbaseOrm.setDatabase(dbTable[0]); hbaseMapping.setDatabase(dbTable[0]);
hbaseOrm.setTable(dbTable[1]); hbaseMapping.setTable(dbTable[1]);
hbaseOrm.setMode(MappingConfig.Mode.PHOENIX); hbaseMapping.setMode(MappingConfig.Mode.PHOENIX);
hbaseOrm.setRowKey(rowKey); hbaseMapping.setRowKey(rowKey);
// 有定义rowKey // 有定义rowKey
if (rowKey != null) { if (rowKey != null) {
MappingConfig.ColumnItem columnItem = new MappingConfig.ColumnItem(); MappingConfig.ColumnItem columnItem = new MappingConfig.ColumnItem();
columnItem.setRowKey(true); columnItem.setRowKey(true);
columnItem.setColumn(rowKey); columnItem.setColumn(rowKey);
hbaseOrm.setRowKeyColumn(columnItem); hbaseMapping.setRowKeyColumn(columnItem);
} }
config.setHbaseOrm(hbaseOrm); config.setHbaseMapping(hbaseMapping);
config.setDataSourceKey(dsKey); config.setDataSourceKey(dsKey);
} else { } else {

72
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -81,9 +81,9 @@ public class HbaseEtlService {
public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) {
try { try {
// 判断hbase表是否存在，不存在则建表 // 判断hbase表是否存在，不存在则建表
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
if (!hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
hbaseTemplate.createTable(hbaseOrm.getHbaseTable(), hbaseOrm.getFamily()); hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily());
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
@@ -113,29 +113,29 @@ public class HbaseEtlService {
etlResult.setErrorMessage("Config is null!"); etlResult.setErrorMessage("Config is null!");
return etlResult; return etlResult;
} }
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
hbaseTable = hbaseOrm.getHbaseTable(); hbaseTable = hbaseMapping.getHbaseTable();
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) {
logger.info(hbaseOrm.getHbaseTable() + " rebuild is starting!"); logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!");
// 如果表存在则删除 // 如果表存在则删除
if (hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) { if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
hbaseTemplate.disableTable(hbaseOrm.getHbaseTable()); hbaseTemplate.disableTable(hbaseMapping.getHbaseTable());
hbaseTemplate.deleteTable(hbaseOrm.getHbaseTable()); hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable());
} }
params = null; params = null;
} else { } else {
logger.info(hbaseOrm.getHbaseTable() + " etl is starting!"); logger.info(hbaseMapping.getHbaseTable() + " etl is starting!");
} }
createTable(hbaseTemplate, config); createTable(hbaseTemplate, config);
// 拼接sql // 拼接sql
String sql = "SELECT * FROM " + config.getHbaseOrm().getDatabase() + "." + hbaseOrm.getTable(); String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable();
// 拼接条件 // 拼接条件
if (params != null && params.size() == 1 && hbaseOrm.getEtlCondition() == null) { if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) {
AtomicBoolean stExists = new AtomicBoolean(false); AtomicBoolean stExists = new AtomicBoolean(false);
// 验证是否有SYS_TIME字段 // 验证是否有SYS_TIME字段
sqlRS(ds, sql, rs -> { sqlRS(ds, sql, rs -> {
@@ -157,8 +157,8 @@ public class HbaseEtlService {
if (stExists.get()) { if (stExists.get()) {
sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; sql += " WHERE SYS_TIME >= '" + params.get(0) + "' ";
} }
} else if (hbaseOrm.getEtlCondition() != null && params != null) { } else if (hbaseMapping.getEtlCondition() != null && params != null) {
String etlCondition = hbaseOrm.getEtlCondition(); String etlCondition = hbaseMapping.getEtlCondition();
int size = params.size(); int size = params.size();
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
@@ -200,7 +200,7 @@ public class HbaseEtlService {
sqlFinal = sql + " LIMIT " + offset + "," + cnt; sqlFinal = sql + " LIMIT " + offset + "," + cnt;
} }
Future<Boolean> future = executor Future<Boolean> future = executor
.submit(() -> executeSqlImport(ds, sqlFinal, hbaseOrm, hbaseTemplate, successCount, errMsg)); .submit(() -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg));
futures.add(future); futures.add(future);
} }
@@ -210,13 +210,13 @@ public class HbaseEtlService {
executor.shutdown(); executor.shutdown();
} else { } else {
executeSqlImport(ds, sql, hbaseOrm, hbaseTemplate, successCount, errMsg); executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg);
} }
logger.info( logger.info(
hbaseOrm.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); hbaseMapping.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
etlResult.setResultMessage("导入HBase表 " + hbaseOrm.getHbaseTable() + " 数据：" + successCount.get() + " 条"); etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条");
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
@@ -235,13 +235,13 @@ public class HbaseEtlService {
* *
* @param ds * @param ds
* @param sql * @param sql
* @param hbaseOrm * @param hbaseMapping
* @param hbaseTemplate * @param hbaseTemplate
* @param successCount * @param successCount
* @param errMsg * @param errMsg
* @return * @return
*/ */
private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm, private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping,
HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) { HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) {
try { try {
sqlRS(ds, sql, rs -> { sqlRS(ds, sql, rs -> {
@@ -251,8 +251,8 @@ public class HbaseEtlService {
boolean complete = false; boolean complete = false;
List<HRow> rows = new ArrayList<>(); List<HRow> rows = new ArrayList<>();
String[] rowKeyColumns = null; String[] rowKeyColumns = null;
if (hbaseOrm.getRowKey() != null) { if (hbaseMapping.getRowKey() != null) {
rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
} }
while (rs.next()) { while (rs.next()) {
int cc = rs.getMetaData().getColumnCount(); int cc = rs.getMetaData().getColumnCount();
@@ -290,30 +290,30 @@ public class HbaseEtlService {
continue; continue;
} }
MappingConfig.ColumnItem columnItem = hbaseOrm.getColumnItems().get(columnName); MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
// 没有配置映射 // 没有配置映射
if (columnItem == null) { if (columnItem == null) {
String family = hbaseOrm.getFamily(); String family = hbaseMapping.getFamily();
String qualifile = columnName; String qualifile = columnName;
if (hbaseOrm.isUppercaseQualifier()) { if (hbaseMapping.isUppercaseQualifier()) {
qualifile = qualifile.toUpperCase(); qualifile = qualifile.toUpperCase();
} }
if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) { if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
if (hbaseOrm.getRowKey() == null && j == 1) { if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(Bytes.toBytes(val.toString())); row.setRowKey(Bytes.toBytes(val.toString()));
} else { } else {
row.addCell(family, qualifile, Bytes.toBytes(val.toString())); row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
} }
} else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) { } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
Type type = Type.getType(classes[j - 1]); Type type = Type.getType(classes[j - 1]);
if (hbaseOrm.getRowKey() == null && j == 1) { if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(TypeUtil.toBytes(val, type)); row.setRowKey(TypeUtil.toBytes(val, type));
} else { } else {
row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
} }
} else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) { } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
PhType phType = PhType.getType(classes[j - 1]); PhType phType = PhType.getType(classes[j - 1]);
if (hbaseOrm.getRowKey() == null && j == 1) { if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(PhTypeUtil.toBytes(val, phType)); row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else { } else {
row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
@@ -353,8 +353,8 @@ public class HbaseEtlService {
rows.add(row); rows.add(row);
complete = false; complete = false;
if (i % hbaseOrm.getCommitBatch() == 0 && !rows.isEmpty()) { if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rows.clear(); rows.clear();
complete = true; complete = true;
} }
@@ -366,12 +366,12 @@ public class HbaseEtlService {
} }
if (!complete && !rows.isEmpty()) { if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage()); errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
// throw new RuntimeException(e); // throw new RuntimeException(e);
} }
return i; return i;

116
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -59,11 +59,11 @@ public class HbaseSyncService {
return; return;
} }
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
// if (!validHTable(config)) { // if (!validHTable(config)) {
// logger.error("HBase table '{}' not exists", // logger.error("HBase table '{}' not exists",
// hbaseOrm.getHbaseTable()); // hbaseMapping.getHbaseTable());
// return; // return;
// } // }
int i = 1; int i = 1;
@@ -73,28 +73,28 @@ public class HbaseSyncService {
HRow hRow = new HRow(); HRow hRow = new HRow();
// 拼接复合rowKey // 拼接复合rowKey
if (hbaseOrm.getRowKey() != null) { if (hbaseMapping.getRowKey() != null) {
String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
String rowKeyVale = getRowKeys(rowKeyColumns, r); String rowKeyVale = getRowKeys(rowKeyColumns, r);
// params.put("rowKey", Bytes.toBytes(rowKeyVale)); // params.put("rowKey", Bytes.toBytes(rowKeyVale));
hRow.setRowKey(Bytes.toBytes(rowKeyVale)); hRow.setRowKey(Bytes.toBytes(rowKeyVale));
} }
convertData2Row(hbaseOrm, hRow, r); convertData2Row(hbaseMapping, hRow, r);
if (hRow.getRowKey() == null) { if (hRow.getRowKey() == null) {
throw new RuntimeException("empty rowKey"); throw new RuntimeException("empty rowKey");
} }
rows.add(hRow); rows.add(hRow);
complete = false; complete = false;
if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rows.clear(); rows.clear();
complete = true; complete = true;
} }
i++; i++;
} }
if (!complete && !rows.isEmpty()) { if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
} }
} }
@@ -102,30 +102,30 @@ public class HbaseSyncService {
/** /**
* 将Map数据转换为HRow行数据 * 将Map数据转换为HRow行数据
* *
* @param hbaseOrm hbase映射配置 * @param hbaseMapping hbase映射配置
* @param hRow 行对象 * @param hRow 行对象
* @param data Map数据 * @param data Map数据
*/ */
private static void convertData2Row(MappingConfig.HbaseOrm hbaseOrm, HRow hRow, Map<String, Object> data) { private static void convertData2Row(MappingConfig.HbaseMapping hbaseMapping, HRow hRow, Map<String, Object> data) {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems(); Map<String, MappingConfig.ColumnItem> columnItems = hbaseMapping.getColumnItems();
int i = 0; int i = 0;
for (Map.Entry<String, Object> entry : data.entrySet()) { for (Map.Entry<String, Object> entry : data.entrySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) { if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(entry.getKey())) {
continue; continue;
} }
if (entry.getValue() != null) { if (entry.getValue() != null) {
MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey());
byte[] bytes = typeConvert(columnItem, hbaseOrm, entry.getValue()); byte[] bytes = typeConvert(columnItem, hbaseMapping, entry.getValue());
if (columnItem == null) { if (columnItem == null) {
String familyName = hbaseOrm.getFamily(); String familyName = hbaseMapping.getFamily();
String qualifier = entry.getKey(); String qualifier = entry.getKey();
if (hbaseOrm.isUppercaseQualifier()) { if (hbaseMapping.isUppercaseQualifier()) {
qualifier = qualifier.toUpperCase(); qualifier = qualifier.toUpperCase();
} }
if (hbaseOrm.getRowKey() == null && i == 0) { if (hbaseMapping.getRowKey() == null && i == 0) {
hRow.setRowKey(bytes); hRow.setRowKey(bytes);
} else { } else {
hRow.addCell(familyName, qualifier, bytes); hRow.addCell(familyName, qualifier, bytes);
@@ -156,15 +156,15 @@ public class HbaseSyncService {
return; return;
} }
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
// if (!validHTable(config)) { // if (!validHTable(config)) {
// logger.error("HBase table '{}' not exists", // logger.error("HBase table '{}' not exists",
// hbaseOrm.getHbaseTable()); // hbaseMapping.getHbaseTable());
// return; // return;
// } // }
MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn(); MappingConfig.ColumnItem rowKeyColumn = hbaseMapping.getRowKeyColumn();
int index = 0; int index = 0;
int i = 1; int i = 1;
boolean complete = false; boolean complete = false;
@@ -172,8 +172,8 @@ public class HbaseSyncService {
out: for (Map<String, Object> r : data) { out: for (Map<String, Object> r : data) {
byte[] rowKeyBytes; byte[] rowKeyBytes;
if (hbaseOrm.getRowKey() != null) { if (hbaseMapping.getRowKey() != null) {
String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
// 判断是否有复合主键修改 // 判断是否有复合主键修改
for (String updateColumn : old.get(index).keySet()) { for (String updateColumn : old.get(index).keySet()) {
@@ -190,23 +190,23 @@ public class HbaseSyncService {
rowKeyBytes = Bytes.toBytes(rowKeyVale); rowKeyBytes = Bytes.toBytes(rowKeyVale);
} else if (rowKeyColumn == null) { } else if (rowKeyColumn == null) {
Map<String, Object> rowKey = data.get(0); Map<String, Object> rowKey = data.get(0);
rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next()); rowKeyBytes = typeConvert(null, hbaseMapping, rowKey.values().iterator().next());
} else { } else {
rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, r.get(rowKeyColumn.getColumn())); rowKeyBytes = typeConvert(rowKeyColumn, hbaseMapping, r.get(rowKeyColumn.getColumn()));
} }
if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空"); if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空");
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems(); Map<String, MappingConfig.ColumnItem> columnItems = hbaseMapping.getColumnItems();
HRow hRow = new HRow(rowKeyBytes); HRow hRow = new HRow(rowKeyBytes);
for (String updateColumn : old.get(index).keySet()) { for (String updateColumn : old.get(index).keySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) { if (hbaseMapping.getExcludeColumns() != null && hbaseMapping.getExcludeColumns().contains(updateColumn)) {
continue; continue;
} }
MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);
if (columnItem == null) { if (columnItem == null) {
String family = hbaseOrm.getFamily(); String family = hbaseMapping.getFamily();
String qualifier = updateColumn; String qualifier = updateColumn;
if (hbaseOrm.isUppercaseQualifier()) { if (hbaseMapping.isUppercaseQualifier()) {
qualifier = qualifier.toUpperCase(); qualifier = qualifier.toUpperCase();
} }
@@ -215,7 +215,7 @@ public class HbaseSyncService {
if (newVal == null) { if (newVal == null) {
hRow.addCell(family, qualifier, null); hRow.addCell(family, qualifier, null);
} else { } else {
hRow.addCell(family, qualifier, typeConvert(null, hbaseOrm, newVal)); hRow.addCell(family, qualifier, typeConvert(null, hbaseMapping, newVal));
} }
} else { } else {
// 排除修改id的情况 // 排除修改id的情况
@@ -227,14 +227,14 @@ public class HbaseSyncService {
} else { } else {
hRow.addCell(columnItem.getFamily(), hRow.addCell(columnItem.getFamily(),
columnItem.getQualifier(), columnItem.getQualifier(),
typeConvert(columnItem, hbaseOrm, newVal)); typeConvert(columnItem, hbaseMapping, newVal));
} }
} }
} }
rows.add(hRow); rows.add(hRow);
complete = false; complete = false;
if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rows.clear(); rows.clear();
complete = true; complete = true;
} }
@@ -242,7 +242,7 @@ public class HbaseSyncService {
index++; index++;
} }
if (!complete && !rows.isEmpty()) { if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
} }
} }
@@ -252,45 +252,45 @@ public class HbaseSyncService {
return; return;
} }
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
// if (!validHTable(config)) { // if (!validHTable(config)) {
// logger.error("HBase table '{}' not exists", // logger.error("HBase table '{}' not exists",
// hbaseOrm.getHbaseTable()); // hbaseMapping.getHbaseTable());
// return; // return;
// } // }
MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn(); MappingConfig.ColumnItem rowKeyColumn = hbaseMapping.getRowKeyColumn();
boolean complete = false; boolean complete = false;
int i = 1; int i = 1;
Set<byte[]> rowKeys = new HashSet<>(); Set<byte[]> rowKeys = new HashSet<>();
for (Map<String, Object> r : data) { for (Map<String, Object> r : data) {
byte[] rowKeyBytes; byte[] rowKeyBytes;
if (hbaseOrm.getRowKey() != null) { if (hbaseMapping.getRowKey() != null) {
String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
String rowKeyVale = getRowKeys(rowKeyColumns, r); String rowKeyVale = getRowKeys(rowKeyColumns, r);
rowKeyBytes = Bytes.toBytes(rowKeyVale); rowKeyBytes = Bytes.toBytes(rowKeyVale);
} else if (rowKeyColumn == null) { } else if (rowKeyColumn == null) {
// 如果不需要类型转换 // 如果不需要类型转换
Map<String, Object> rowKey = data.get(0); Map<String, Object> rowKey = data.get(0);
rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next()); rowKeyBytes = typeConvert(null, hbaseMapping, rowKey.values().iterator().next());
} else { } else {
Object val = r.get(rowKeyColumn.getColumn()); Object val = r.get(rowKeyColumn.getColumn());
rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, val); rowKeyBytes = typeConvert(rowKeyColumn, hbaseMapping, val);
} }
if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空"); if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空");
rowKeys.add(rowKeyBytes); rowKeys.add(rowKeyBytes);
complete = false; complete = false;
if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rowKeys.isEmpty()) { if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rowKeys.isEmpty()) {
hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys);
rowKeys.clear(); rowKeys.clear();
complete = true; complete = true;
} }
i++; i++;
} }
if (!complete && !rowKeys.isEmpty()) { if (!complete && !rowKeys.isEmpty()) {
hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys);
} }
} }
@@ -300,9 +300,9 @@ public class HbaseSyncService {
if (old == null || old.isEmpty() || data == null || data.isEmpty()) { if (old == null || old.isEmpty() || data == null || data.isEmpty()) {
return; return;
} }
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(","); String[] rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
int index = 0; int index = 0;
int i = 1; int i = 1;
@@ -343,13 +343,13 @@ public class HbaseSyncService {
rowKeys.add(oldRowKeyBytes); rowKeys.add(oldRowKeyBytes);
HRow row = new HRow(newRowKeyBytes); HRow row = new HRow(newRowKeyBytes);
convertData2Row(hbaseOrm, row, r); convertData2Row(hbaseMapping, row, r);
rows.add(row); rows.add(row);
complete = false; complete = false;
if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) { if (i % config.getHbaseMapping().getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys);
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rowKeys.clear(); rowKeys.clear();
rows.clear(); rows.clear();
complete = true; complete = true;
@@ -358,8 +358,8 @@ public class HbaseSyncService {
index++; index++;
} }
if (!complete && !rows.isEmpty()) { if (!complete && !rows.isEmpty()) {
hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys); hbaseTemplate.deletes(hbaseMapping.getHbaseTable(), rowKeys);
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
} }
} }
@@ -367,32 +367,32 @@ public class HbaseSyncService {
* 根据对应的类型进行转换 * 根据对应的类型进行转换
* *
* @param columnItem 列项配置 * @param columnItem 列项配置
* @param hbaseOrm hbase映射配置 * @param hbaseMapping hbase映射配置
* @param value 值 * @param value 值
* @return 复合字段rowKey * @return 复合字段rowKey
*/ */
private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm, private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseMapping hbaseMapping,
Object value) { Object value) {
if (value == null) { if (value == null) {
return null; return null;
} }
byte[] bytes = null; byte[] bytes = null;
if (columnItem == null || columnItem.getType() == null || "".equals(columnItem.getType())) { if (columnItem == null || columnItem.getType() == null || "".equals(columnItem.getType())) {
if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) { if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
bytes = Bytes.toBytes(value.toString()); bytes = Bytes.toBytes(value.toString());
} else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) { } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
bytes = TypeUtil.toBytes(value); bytes = TypeUtil.toBytes(value);
} else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) { } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
PhType phType = PhType.getType(value.getClass()); PhType phType = PhType.getType(value.getClass());
bytes = PhTypeUtil.toBytes(value, phType); bytes = PhTypeUtil.toBytes(value, phType);
} }
} else { } else {
if (hbaseOrm.getMode() == MappingConfig.Mode.STRING) { if (hbaseMapping.getMode() == MappingConfig.Mode.STRING) {
bytes = Bytes.toBytes(value.toString()); bytes = Bytes.toBytes(value.toString());
} else if (hbaseOrm.getMode() == MappingConfig.Mode.NATIVE) { } else if (hbaseMapping.getMode() == MappingConfig.Mode.NATIVE) {
Type type = Type.getType(columnItem.getType()); Type type = Type.getType(columnItem.getType());
bytes = TypeUtil.toBytes(value, type); bytes = TypeUtil.toBytes(value, type);
} else if (hbaseOrm.getMode() == MappingConfig.Mode.PHOENIX) { } else if (hbaseMapping.getMode() == MappingConfig.Mode.PHOENIX) {
PhType phType = PhType.getType(columnItem.getType()); PhType phType = PhType.getType(columnItem.getType());
bytes = PhTypeUtil.toBytes(value, phType); bytes = PhTypeUtil.toBytes(value, phType);
} }

4
client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml
View File

@@ -1,5 +1,5 @@
dataSourceKey: defaultDS dataSourceKey: defaultDS
hbaseOrm: hbaseMapping:
mode: PHOENIX #NATIVE #STRING mode: PHOENIX #NATIVE #STRING
destination: example destination: example
database: mytest # 数据库名 database: mytest # 数据库名
@@ -18,7 +18,7 @@ hbaseOrm:
c_time: C_TIME$UNSIGNED_TIMESTAMP c_time: C_TIME$UNSIGNED_TIMESTAMP
birthday: BIRTHDAY$DATE birthday: BIRTHDAY$DATE
excludeColumns: excludeColumns:
- lat - lat # 忽略字段
# -- NATIVE类型 # -- NATIVE类型
# $DEFAULT # $DEFAULT

32
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -10,9 +10,9 @@ spring:
default-property-inclusion: non_null default-property-inclusion: non_null
hbase.zookeeper.quorum: slave1 hbasezookeeper.quorum: 127.0.0.1
hbase.zookeeper.property.clientPort: 2181 hbase.zookeeper.property.clientPort: 2181
hbase.zookeeper.znode.parent: /hbase-unsecure hbase.zookeeper.znode.parent: /hbase
canal.conf: canal.conf:
canalServerHost: 127.0.0.1:11111 canalServerHost: 127.0.0.1:11111
@@ -23,12 +23,12 @@ canal.conf:
- instance: example - instance: example
adapterGroups: adapterGroups:
- outAdapters: - outAdapters:
# - name: logger - name: logger
- name: hbase # - name: hbase
properties: # properties:
hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} # hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# mqTopics: # mqTopics:
# - mqMode: kafka # - mqMode: kafka
# topic: example # topic: example
@@ -37,11 +37,11 @@ canal.conf:
# outAdapters: # outAdapters:
# - name: logger # - name: logger
adapter.conf: #adapter.conf:
datasourceConfigs: # datasourceConfigs:
defaultDS: # defaultDS:
url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true # url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
username: root # username: root
password: 121212 # password: 121212
adapterConfigs: # adapterConfigs:
- hbase/mytest_person2.yml # - hbase/mytest_person2.yml