From 0d0fcbbc0b885d577ebe08eed33ae1b9e413e0fc Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 26 Oct 2018 14:03:29 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9HBase=E9=80=82=E9=85=8D?= =?UTF-8?q?=E5=99=A8=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/hbase/config/MappingConfig.java | 9 ++++++ .../hbase/service/HbaseSyncService.java | 6 ++++ .../main/resources/hbase/mytest_person2.yml | 2 ++ .../src/main/resources/application.yml | 28 +++++++++---------- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 8027fb51..93968610 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -152,6 +152,7 @@ public class MappingConfig { private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 private String rowKey; // 指定复合主键为rowKey private Map columns; // 字段映射 + private List excludeColumns; // 不映射的字段 private ColumnItem rowKeyColumn; // rowKey字段 private String etlCondition; // etl条件sql @@ -292,6 +293,14 @@ public class MappingConfig { } } + public List getExcludeColumns() { + return excludeColumns; + } + + public void setExcludeColumns(List excludeColumns) { + this.excludeColumns = excludeColumns; + } + public String getFamily() { return family; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java index 2edc396b..4f31d0d3 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java @@ -110,6 +110,9 @@ public class HbaseSyncService { Map columnItems = hbaseOrm.getColumnItems(); int i = 0; for (Map.Entry entry : data.entrySet()) { + if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) { + continue; + } if (entry.getValue() != null) { MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); @@ -196,6 +199,9 @@ public class HbaseSyncService { Map columnItems = hbaseOrm.getColumnItems(); HRow hRow = new HRow(rowKeyBytes); for (String updateColumn : old.get(index).keySet()) { + if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) { + continue; + } MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); if (columnItem == null) { String family = hbaseOrm.getFamily(); diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index b22ffe5c..d80b95eb 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -17,6 +17,8 @@ hbaseOrm: type: $DECIMAL c_time: C_TIME$UNSIGNED_TIMESTAMP birthday: BIRTHDAY$DATE + excludeColumns: + - lat # -- NATIVE类型 # $DEFAULT diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index 43cc4e38..1b1139c1 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -23,12 +23,12 @@ canal.conf: - instance: example adapterGroups: - outAdapters: - - name: logger -# - name: hbase -# properties: -# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} -# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} -# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} +# - name: logger + - name: hbase + properties: + hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} + hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} + zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # mqTopics: # - mqMode: kafka # topic: example @@ -37,11 +37,11 @@ canal.conf: # outAdapters: # - name: logger -#adapter.conf: -# datasourceConfigs: -# defaultDS: -# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true -# username: root -# password: 121212 -# adapterConfigs: -# - hbase/mytest_person2.yml +adapter.conf: + datasourceConfigs: + defaultDS: + url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true + username: root + password: 121212 + adapterConfigs: + - hbase/mytest_person2.yml From 0d22dbc9828e24cc5ffbf8bd77914db8b93b990a Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 26 Oct 2018 15:56:05 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=89=93=E5=8C=85?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/hbase/pom.xml | 78 ++++++++++----------------------- client-adapter/launcher/pom.xml | 5 +++ client-adapter/logger/pom.xml | 57 ++++++++++-------------- 3 files changed, 51 insertions(+), 89 deletions(-) diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index f48cdd10..b3d274e4 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -43,60 +43,28 @@ + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + - - - release - - - env - release - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - maven-antrun-plugin - - - package - - run - - - - - - - - - - - - - - - - - diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index f10dfaac..8159bb66 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -139,6 +139,11 @@ + + + + + diff --git a/client-adapter/logger/pom.xml b/client-adapter/logger/pom.xml index e9a146aa..af7ec379 100644 --- a/client-adapter/logger/pom.xml +++ b/client-adapter/logger/pom.xml @@ -21,39 +21,28 @@ - - - release - - - env - release - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - - - + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + +