chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

HBase的etl初步完成

Browse Source
This commit is contained in:
mcy
2018-10-24 18:30:42 +08:00
parent 15e79c1400
commit 0a7251add0
17 changed files with 869 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java
View File

@@ -1,8 +1,8 @@
package com.alibaba.otter.canal.client.adapter;
import java.util.Map;
import java.util.List;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.SPI;
@@ -20,7 +20,7 @@ public interface OuterAdapter {
*
* @param configuration 外部适配器配置信息
*/
void init(CanalOuterAdapterConfiguration configuration);
void init(OuterAdapterConfig configuration);
/**
* 往适配器中同步数据
@@ -29,8 +29,6 @@ public interface OuterAdapter {
*/
void sync(Dml dml);
// void writeOut(FlatMessage flatMessage);
/**
* 外部适配器销毁接口
*/
@@ -39,9 +37,10 @@ public interface OuterAdapter {
/**
* Etl操作
*
* @param criteriaSql 条件拼接sql
* @param task 任务名, 对应配置名
* @param params etl筛选条件
*/
default void etl(String criteriaSql) {
default void etl(String task, List<String> params) {
throw new UnsupportedOperationException();
}

12
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -109,13 +109,13 @@ public class CanalClientConfig {
public static class AdapterGroup {
private List<CanalOuterAdapterConfiguration> outAdapters;
private List<OuterAdapterConfig> outAdapters;
public List<CanalOuterAdapterConfiguration> getOutAdapters() {
public List<OuterAdapterConfig> getOutAdapters() {
return outAdapters;
}
public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters) {
public void setOutAdapters(List<OuterAdapterConfig> outAdapters) {
this.outAdapters = outAdapters;
}
}
@@ -159,7 +159,7 @@ public class CanalClientConfig {
// private List<Adaptor> adapters = new ArrayList<>();
private List<CanalOuterAdapterConfiguration> outAdapters;
private List<OuterAdapterConfig> outAdapters;
public String getGroupId() {
return groupId;
@@ -169,11 +169,11 @@ public class CanalClientConfig {
this.groupId = groupId;
}
public List<CanalOuterAdapterConfiguration> getOutAdapters() {
public List<OuterAdapterConfig> getOutAdapters() {
return outAdapters;
}
public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters) {
public void setOutAdapters(List<OuterAdapterConfig> outAdapters) {
this.outAdapters = outAdapters;
}

54
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java
View File

@@ -2,10 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.sql.*;
import org.joda.time.DateTime;
import org.slf4j.Logger;
@@ -21,6 +18,55 @@ public class JdbcTypeUtil {
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
return rs.getByte(columnName);
} else {
return rs.getObject(columnName);
}
}
public static Class<?> jdbcType2javaType(int jdbcType) {
switch (jdbcType) {
case Types.BIT:
case Types.BOOLEAN:
// return Boolean.class;
case Types.TINYINT:
return Byte.TYPE;
case Types.SMALLINT:
return Short.class;
case Types.INTEGER:
return Integer.class;
case Types.BIGINT:
return Long.class;
case Types.DECIMAL:
case Types.NUMERIC:
return BigDecimal.class;
case Types.REAL:
return Float.class;
case Types.FLOAT:
case Types.DOUBLE:
return Double.class;
case Types.CHAR:
case Types.VARCHAR:
case Types.LONGVARCHAR:
return String.class;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
return byte[].class;
case Types.DATE:
return java.sql.Date.class;
case Types.TIME:
return Time.class;
case Types.TIMESTAMP:
return Timestamp.class;
default:
return String.class;
}
}
public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) {
if (value == null || value.equals("")) {
return null;

16
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java → client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/OuterAdapterConfig.java
View File

@@ -1,6 +1,6 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.util.Properties;
import java.util.Map;
/**
* 外部适配器配置信息类
@@ -8,15 +8,15 @@ import java.util.Properties;
* @author machengyuan 2018-8-18 下午10:15:12
* @version 1.0.0
*/
public class CanalOuterAdapterConfiguration {
public class OuterAdapterConfig {
private String name; // 适配器名称, 如: logger, hbase, es
private String name; // 适配器名称, 如: logger, hbase, es
private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址
private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址
private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址
private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址
private Properties properties; // 其余参数, 可填写适配器中的所需的配置信息
private Map<String, String> properties; // 其余参数, 可填写适配器中的所需的配置信息
public String getName() {
return name;
@@ -34,11 +34,11 @@ public class CanalOuterAdapterConfiguration {
this.hosts = hosts;
}
public Properties getProperties() {
public Map<String, String> getProperties() {
return properties;
}
public void setProperties(Properties properties) {
public void setProperties(Map<String, String> properties) {
this.properties = properties;
}