chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1045 from rewerma/master

Browse Source 
修改打包配置
This commit is contained in:
rewerma
2018-10-26 15:57:07 +08:00
committed by GitHub
parent fb936f896e 0d22dbc982
commit 30b7de7233
7 changed files with 82 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
client-adapter/hbase/pom.xml
View File

@@ -43,60 +43,28 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../launcher/target/config/hbase" overwrite="true" >
<fileset dir="${project.basedir}/src/main/resources/hbase" erroronmissingdir="true">
<include name="*.yml"/>
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

9
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -152,6 +152,7 @@ public class MappingConfig {
private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表
private String rowKey; // 指定复合主键为rowKey
private Map<String, String> columns; // 字段映射
private List<String> excludeColumns; // 不映射的字段
private ColumnItem rowKeyColumn; // rowKey字段
private String etlCondition; // etl条件sql
@@ -292,6 +293,14 @@ public class MappingConfig {
}
}
public List<String> getExcludeColumns() {
return excludeColumns;
}
public void setExcludeColumns(List<String> excludeColumns) {
this.excludeColumns = excludeColumns;
}
public String getFamily() {
return family;
}

6
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -110,6 +110,9 @@ public class HbaseSyncService {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
int i = 0;
for (Map.Entry<String, Object> entry : data.entrySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) {
continue;
}
if (entry.getValue() != null) {
MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey());
@@ -196,6 +199,9 @@ public class HbaseSyncService {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
HRow hRow = new HRow(rowKeyBytes);
for (String updateColumn : old.get(index).keySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) {
continue;
}
MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);
if (columnItem == null) {
String family = hbaseOrm.getFamily();

2
client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml
View File

@@ -17,6 +17,8 @@ hbaseOrm:
type: $DECIMAL
c_time: C_TIME$UNSIGNED_TIMESTAMP
birthday: BIRTHDAY$DATE
excludeColumns:
- lat
# -- NATIVE类型
# $DEFAULT

5
client-adapter/launcher/pom.xml
View File

@@ -139,6 +139,11 @@
<include name="*.yml"/>
</fileset>
</copy>
<copy todir="${project.basedir}/target/config/hbase" overwrite="true" >
<fileset dir="${project.basedir}/../hbase/src/main/resources/hbase" erroronmissingdir="true">
<include name="*.yml"/>
</fileset>
</copy>
</tasks>
</configuration>
</execution>

28
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -23,12 +23,12 @@ canal.conf:
- instance: example
adapterGroups:
- outAdapters:
- name: logger
# - name: hbase
# properties:
# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# - name: logger
- name: hbase
properties:
hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# mqTopics:
# - mqMode: kafka
# topic: example
@@ -37,11 +37,11 @@ canal.conf:
# outAdapters:
# - name: logger
#adapter.conf:
# datasourceConfigs:
# defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
# username: root
# password: 121212
# adapterConfigs:
# - hbase/mytest_person2.yml
adapter.conf:
datasourceConfigs:
defaultDS:
url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
username: root
password: 121212
adapterConfigs:
- hbase/mytest_person2.yml

57
client-adapter/logger/pom.xml
View File

@@ -21,39 +21,28 @@
</dependency>
</dependencies>
<profiles>
<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>