chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1045 from rewerma/master

Browse Source 
修改打包配置
This commit is contained in:
rewerma
2018-10-26 15:57:07 +08:00
committed by GitHub
parent fb936f896e 0d22dbc982
commit 30b7de7233
7 changed files with 82 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
client-adapter/hbase/pom.xml
View File

@@ -43,60 +43,28 @@
</dependency> </dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/../launcher/target/config/hbase" overwrite="true" >
<fileset dir="${project.basedir}/src/main/resources/hbase" erroronmissingdir="true">
<include name="*.yml"/>
</fileset>
</copy>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project> </project>

9
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -152,6 +152,7 @@ public class MappingConfig {
private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表 private boolean autoCreateTable = false; // 同步时HBase中表不存在的情况下自动建表
private String rowKey; // 指定复合主键为rowKey private String rowKey; // 指定复合主键为rowKey
private Map<String, String> columns; // 字段映射 private Map<String, String> columns; // 字段映射
private List<String> excludeColumns; // 不映射的字段
private ColumnItem rowKeyColumn; // rowKey字段 private ColumnItem rowKeyColumn; // rowKey字段
private String etlCondition; // etl条件sql private String etlCondition; // etl条件sql
@@ -292,6 +293,14 @@ public class MappingConfig {
} }
} }
public List<String> getExcludeColumns() {
return excludeColumns;
}
public void setExcludeColumns(List<String> excludeColumns) {
this.excludeColumns = excludeColumns;
}
public String getFamily() { public String getFamily() {
return family; return family;
} }

6
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -110,6 +110,9 @@ public class HbaseSyncService {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems(); Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
int i = 0; int i = 0;
for (Map.Entry<String, Object> entry : data.entrySet()) { for (Map.Entry<String, Object> entry : data.entrySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) {
continue;
}
if (entry.getValue() != null) { if (entry.getValue() != null) {
MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey()); MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey());
@@ -196,6 +199,9 @@ public class HbaseSyncService {
Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems(); Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
HRow hRow = new HRow(rowKeyBytes); HRow hRow = new HRow(rowKeyBytes);
for (String updateColumn : old.get(index).keySet()) { for (String updateColumn : old.get(index).keySet()) {
if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) {
continue;
}
MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn); MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);
if (columnItem == null) { if (columnItem == null) {
String family = hbaseOrm.getFamily(); String family = hbaseOrm.getFamily();

2
client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml
View File

@@ -17,6 +17,8 @@ hbaseOrm:
type: $DECIMAL type: $DECIMAL
c_time: C_TIME$UNSIGNED_TIMESTAMP c_time: C_TIME$UNSIGNED_TIMESTAMP
birthday: BIRTHDAY$DATE birthday: BIRTHDAY$DATE
excludeColumns:
- lat
# -- NATIVE类型 # -- NATIVE类型
# $DEFAULT # $DEFAULT

5
client-adapter/launcher/pom.xml
View File

@@ -139,6 +139,11 @@
<include name="*.yml"/> <include name="*.yml"/>
</fileset> </fileset>
</copy> </copy>
<copy todir="${project.basedir}/target/config/hbase" overwrite="true" >
<fileset dir="${project.basedir}/../hbase/src/main/resources/hbase" erroronmissingdir="true">
<include name="*.yml"/>
</fileset>
</copy>
</tasks> </tasks>
</configuration> </configuration>
</execution> </execution>

28
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -23,12 +23,12 @@ canal.conf:
- instance: example - instance: example
adapterGroups: adapterGroups:
- outAdapters: - outAdapters:
- name: logger # - name: logger
# - name: hbase - name: hbase
# properties: properties:
# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# mqTopics: # mqTopics:
# - mqMode: kafka # - mqMode: kafka
# topic: example # topic: example
@@ -37,11 +37,11 @@ canal.conf:
# outAdapters: # outAdapters:
# - name: logger # - name: logger
#adapter.conf: adapter.conf:
# datasourceConfigs: datasourceConfigs:
# defaultDS: defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
# username: root username: root
# password: 121212 password: 121212
# adapterConfigs: adapterConfigs:
# - hbase/mytest_person2.yml - hbase/mytest_person2.yml

57
client-adapter/logger/pom.xml
View File

@@ -21,39 +21,28 @@
</dependency> </dependency>
</dependencies> </dependencies>
<profiles>
<profile>
<id>release</id>
<activation>
<property>
<name>env</name>
<value>release</value>
</property>
</activation>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId> <artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version> <version>2.4</version>
<configuration> <configuration>
<descriptorRefs> <descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef> <descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs> </descriptorRefs>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
<id>make-assembly</id> <id>make-assembly</id>
<phase>package</phase> <phase>package</phase>
<goals> <goals>
<goal>single</goal> <goal>single</goal>
</goals> </goals>
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</profile>
</profiles>
</project> </project>