chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

修改HBase适配器配置

Browse Source
This commit is contained in:
mcy
2018-10-26 14:03:29 +08:00
parent bbfd4b4996
commit 0d0fcbbc0b
4 changed files with 31 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -152,6 +152,7 @@ public class MappingConfig {
private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表
private String rowKey; // 指定复合主键为rowKey private String rowKey; // 指定复合主键为rowKey
private Map<String, String> columns; // 字段映射 private Map<String, String> columns; // 字段映射
private List<String> excludeColumns; // 不映射的字段
private ColumnItem rowKeyColumn; // rowKey字段 private ColumnItem rowKeyColumn; // rowKey字段
private String etlCondition; // etl条件sql private String etlCondition; // etl条件sql
@@ -292,6 +293,14 @@ public class MappingConfig {
} }
} }
public List<String> getExcludeColumns() {
return excludeColumns;
}
public void setExcludeColumns(List<String> excludeColumns) {
this.excludeColumns = excludeColumns;
}
public String getFamily() { public String getFamily() {
return family; return family;
} }

6
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -110,6 +110,9 @@ public class HbaseSyncService {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems(); Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
int i = 0; int i = 0;
for (Map.Entry<String, Object> entry : data.entrySet()) { for (Map.Entry<String, Object> entry : data.entrySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) {
continue;
}
if (entry.getValue() != null) { if (entry.getValue() != null) {
MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey());
@@ -196,6 +199,9 @@ public class HbaseSyncService {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems(); Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
HRow hRow = new HRow(rowKeyBytes); HRow hRow = new HRow(rowKeyBytes);
for (String updateColumn : old.get(index).keySet()) { for (String updateColumn : old.get(index).keySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) {
continue;
}
MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);
if (columnItem == null) { if (columnItem == null) {
String family = hbaseOrm.getFamily(); String family = hbaseOrm.getFamily();

2
client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml
View File

@@ -17,6 +17,8 @@ hbaseOrm:
type: $DECIMAL type: $DECIMAL
c_time: C_TIME$UNSIGNED_TIMESTAMP c_time: C_TIME$UNSIGNED_TIMESTAMP
birthday: BIRTHDAY$DATE birthday: BIRTHDAY$DATE
excludeColumns:
- lat
# -- NATIVE类型 # -- NATIVE类型
# $DEFAULT # $DEFAULT

28
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -23,12 +23,12 @@ canal.conf:
- instance: example - instance: example
adapterGroups: adapterGroups:
- outAdapters: - outAdapters:
- name: logger # - name: logger
# - name: hbase - name: hbase
# properties: properties:
# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# mqTopics: # mqTopics:
# - mqMode: kafka # - mqMode: kafka
# topic: example # topic: example
@@ -37,11 +37,11 @@ canal.conf:
# outAdapters: # outAdapters:
# - name: logger # - name: logger
#adapter.conf: adapter.conf:
# datasourceConfigs: datasourceConfigs:
# defaultDS: defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
# username: root username: root
# password: 121212 password: 121212
# adapterConfigs: adapterConfigs:
# - hbase/mytest_person2.yml - hbase/mytest_person2.yml