chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

HBase的etl初步完成

Browse Source
This commit is contained in:
mcy
2018-10-24 18:30:42 +08:00
parent 15e79c1400
commit 0a7251add0
17 changed files with 869 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java
View File

@@ -2,9 +2,11 @@ package com.alibaba.otter.canal.client.adapter.hbase;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import javax.sql.DataSource;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
@@ -13,9 +15,12 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader;
import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService;
import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
/**
@@ -27,56 +32,37 @@ import com.alibaba.otter.canal.client.adapter.support.SPI;
@SPI("hbase")
public class HbaseAdapter implements OuterAdapter {
private static volatile Map<String, MappingConfig> mappingConfigCache = null;
private static volatile Map<String, MappingConfig> hbaseMapping = null; // 文件名对应配置
private static volatile Map<String, MappingConfig> mappingConfigCache = null; // 库名-表名对应配置
private Connection conn;
private HbaseSyncService hbaseSyncService;
private HbaseTemplate hbaseTemplate;
@Override
public void init(CanalOuterAdapterConfiguration configuration) {
public void init(OuterAdapterConfig configuration) {
try {
if (mappingConfigCache == null) {
synchronized (MappingConfig.class) {
if (mappingConfigCache == null) {
Map<String, MappingConfig> hbaseMapping = MappingConfigLoader.load();
hbaseMapping = MappingConfigLoader.load();
mappingConfigCache = new HashMap<>();
for (MappingConfig mappingConfig : hbaseMapping.values()) {
mappingConfigCache.put(mappingConfig.getHbaseOrm().getDatabase() + "-"
+ mappingConfig.getHbaseOrm().getTable(), mappingConfig);
+ mappingConfig.getHbaseOrm().getTable(),
mappingConfig);
}
}
}
}
String hosts = configuration.getZkHosts();
if (StringUtils.isEmpty(hosts)) {
hosts = configuration.getHosts();
}
if (StringUtils.isEmpty(hosts)) {
throw new RuntimeException("Empty zookeeper hosts");
}
String[] zkHosts = StringUtils.split(hosts, ",");
int zkPort = 0;
StringBuilder hostsWithoutPort = new StringBuilder();
for (String host : zkHosts) {
int i = host.indexOf(":");
hostsWithoutPort.append(host, 0, i);
hostsWithoutPort.append(",");
if (zkPort == 0) zkPort = Integer.parseInt(host.substring(i + 1));
}
hostsWithoutPort.deleteCharAt(hostsWithoutPort.length() - 1);
String znode = configuration.getProperties().getProperty("znodeParent");
if (StringUtils.isEmpty(znode)) {
znode = "/hbase";
}
Map<String, String> propertites = configuration.getProperties();
Configuration hbaseConfig = HBaseConfiguration.create();
hbaseConfig.set("hbase.zookeeper.quorum", hostsWithoutPort.toString());
hbaseConfig.set("hbase.zookeeper.property.clientPort", Integer.toString(zkPort));
hbaseConfig.set("zookeeper.znode.parent", znode);
propertites.forEach(hbaseConfig::set);
conn = ConnectionFactory.createConnection(hbaseConfig);
hbaseSyncService = new HbaseSyncService(conn);
hbaseTemplate = new HbaseTemplate(conn);
hbaseSyncService = new HbaseSyncService(hbaseTemplate);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -93,6 +79,15 @@ public class HbaseAdapter implements OuterAdapter {
hbaseSyncService.sync(config, dml);
}
@Override
public void etl(String task, List<String> params) {
MappingConfig config = hbaseMapping.get(task);
DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
if (dataSource != null) {
HbaseEtlService.importData(dataSource, hbaseTemplate, config, params);
}
}
@Override
public void destroy() {
if (conn != null) {

347
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -1,4 +1,351 @@
package com.alibaba.otter.canal.client.adapter.hbase.service;
import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import javax.sql.DataSource;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.support.*;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.google.common.base.Joiner;
public class HbaseEtlService {
private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class);
public static Object sqlRS(DataSource ds, String sql, Function<ResultSet, Object> fun) throws SQLException {
Connection conn = null;
Statement smt = null;
ResultSet rs = null;
try {
conn = ds.getConnection();
smt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
smt.setFetchSize(Integer.MIN_VALUE);
rs = smt.executeQuery(sql);
return fun.apply(rs);
} finally {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
}
if (smt != null) {
try {
smt.close();
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
}
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
}
}
}
public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) {
try {
// 判断hbase表是否存在，不存在则建表
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm();
if (!hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) {
hbaseTemplate.createTable(hbaseOrm.getHbaseTable(), hbaseOrm.getFamilies().toArray(new String[0]));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
public static void importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config,
List<String> params) {
// EtlResult etlResult = new EtlResult();
AtomicLong successCount = new AtomicLong();
List<String> errMsg = new ArrayList<>();
String hbaseTable = "";
try {
// if (config == null) {
// logger.error("Config is null!");
// etlResult.setSucceeded(false);
// etlResult.setErrorMessage("Config is null!");
// return etlResult;
// }
MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm();
hbaseTable = hbaseOrm.getHbaseTable();
long start = System.currentTimeMillis();
if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) {
logger.info(hbaseOrm.getHbaseTable() + " rebuild is starting!");
// 如果表存在则删除
if (hbaseTemplate.tableExists(hbaseOrm.getHbaseTable())) {
hbaseTemplate.disableTable(hbaseOrm.getHbaseTable());
hbaseTemplate.deleteTable(hbaseOrm.getHbaseTable());
}
params = null;
} else {
logger.info(hbaseOrm.getHbaseTable() + " etl is starting!");
}
createTable(hbaseTemplate, config);
// 拼接sql
String sql = "SELECT * FROM " + config.getHbaseOrm().getDatabase() + "." + hbaseOrm.getTable();
// 拼接条件
if (params != null && params.size() == 1 && hbaseOrm.getEtlCondition() == null) {
AtomicBoolean stExists = new AtomicBoolean(false);
// 验证是否有SYS_TIME字段
sqlRS(ds, sql, rs -> {
try {
ResultSetMetaData rsmd = rs.getMetaData();
int cnt = rsmd.getColumnCount();
for (int i = 1; i <= cnt; i++) {
String columnName = rsmd.getColumnName(i);
if ("SYS_TIME".equalsIgnoreCase(columnName)) {
stExists.set(true);
break;
}
}
} catch (Exception e) {
// ignore
}
return null;
});
if (stExists.get()) {
sql += " WHERE SYS_TIME >= '" + params.get(0) + "' ";
}
} else if (hbaseOrm.getEtlCondition() != null && params != null) {
String etlCondition = hbaseOrm.getEtlCondition();
int size = params.size();
for (int i = 0; i < size; i++) {
etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
}
sql += " " + etlCondition;
}
// 获取总数
String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
long cnt = (Long) sqlRS(ds, countSql, rs -> {
Long count = null;
try {
if (rs.next()) {
count = ((Number) rs.getObject(1)).longValue();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return count == null ? 0 : count;
});
// 当大于1万条记录时开启多线程
if (cnt >= 10000) {
int threadCount = 3; // TODO 从配置读取默认为3
long perThreadCnt = cnt / threadCount;
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
List<Future<Boolean>> futures = new ArrayList<>(threadCount);
for (int i = 0; i < threadCount; i++) {
long offset = i * perThreadCnt;
Long size = null;
if (i != threadCount - 1) {
size = perThreadCnt;
}
String sqlFinal;
if (size != null) {
sqlFinal = sql + " LIMIT " + offset + "," + size;
} else {
sqlFinal = sql + " LIMIT " + offset + "," + cnt;
}
Future<Boolean> future = executor
.submit(() -> executeSqlImport(ds, sqlFinal, hbaseOrm, hbaseTemplate, successCount, errMsg));
futures.add(future);
}
for (Future<Boolean> future : futures) {
future.get();
}
executor.shutdown();
} else {
executeSqlImport(ds, sql, hbaseOrm, hbaseTemplate, successCount, errMsg);
}
logger.info(
hbaseOrm.getHbaseTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
// etlResult.setResultMessage("导入HBase表 " + hbaseOrm.getHbaseTable() + " 数据：" +
// successCount.get() + " 条");
} catch (Exception e) {
logger.error(e.getMessage(), e);
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
}
// if (errMsg.isEmpty()) {
// etlResult.setSucceeded(true);
// } else {
// etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
// }
// return etlResult;
}
private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseOrm hbaseOrm,
HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) {
try {
sqlRS(ds, sql, rs -> {
int i = 1;
try {
boolean complete = false;
List<HRow> rows = new ArrayList<>();
String[] rowKeyColumns = null;
if (hbaseOrm.getRowKey() != null) {
rowKeyColumns = hbaseOrm.getRowKey().trim().split(",");
}
while (rs.next()) {
int cc = rs.getMetaData().getColumnCount();
int[] jdbcTypes = new int[cc];
Class<?>[] classes = new Class[cc];
for (int j = 1; j <= cc; j++) {
int jdbcType = rs.getMetaData().getColumnType(j);
jdbcTypes[j - 1] = jdbcType;
classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType);
}
HRow row = new HRow();
if (rowKeyColumns != null) {
// 取rowKey字段拼接
StringBuilder rowKeyVale = new StringBuilder();
for (String rowKeyColumnName : rowKeyColumns) {
Object obj = rs.getObject(rowKeyColumnName);
if (obj != null) {
rowKeyVale.append(obj.toString());
}
rowKeyVale.append("|");
}
int len = rowKeyVale.length();
if (len > 0) {
rowKeyVale.delete(len - 1, len);
}
row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
}
for (int j = 1; j <= cc; j++) {
String columnName = rs.getMetaData().getColumnName(j);
Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
if (val == null) {
continue;
}
MappingConfig.ColumnItem columnItem = hbaseOrm.getColumnItems().get(columnName);
// 没有配置映射
if (columnItem == null) {
String family = hbaseOrm.getFamily();
String qualifile = columnName;
if (hbaseOrm.isUppercaseQualifier()) {
qualifile = qualifile.toUpperCase();
}
if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) {
if (hbaseOrm.getRowKey() == null && j == 1) {
row.setRowKey(Bytes.toBytes(val.toString()));
} else {
row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
}
} else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) {
Type type = Type.getType(classes[j - 1]);
if (hbaseOrm.getRowKey() == null && j == 1) {
row.setRowKey(TypeUtil.toBytes(val, type));
} else {
row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
}
} else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) {
PhType phType = PhType.getType(classes[j - 1]);
if (hbaseOrm.getRowKey() == null && j == 1) {
row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else {
row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
}
}
} else {
// 如果不需要类型转换
if (columnItem.getType() == null || "".equals(columnItem.getType())) {
if (val instanceof java.sql.Date) {
SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
val = dateFmt.format((Date) val);
} else if (val instanceof Timestamp) {
SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
val = datetimeFmt.format((Date) val);
}
byte[] valBytes = Bytes.toBytes(val.toString());
if (columnItem.isRowKey()) {
row.setRowKey(valBytes);
} else {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
}
} else {
PhType phType = PhType.getType(columnItem.getType());
if (columnItem.isRowKey()) {
row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else {
row.addCell(columnItem.getFamily(),
columnItem.getQualifier(),
PhTypeUtil.toBytes(val, phType));
}
}
}
}
if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
rows.add(row);
complete = false;
if (i % hbaseOrm.getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
rows.clear();
complete = true;
}
if (logger.isDebugEnabled()) {
logger.debug("import count:" + i);
}
// System.out.println(i);
i++;
successCount.incrementAndGet();
}
if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
}
} catch (Exception e) {
logger.error(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(hbaseOrm.getHbaseTable() + " etl failed! ==>" + e.getMessage());
throw new RuntimeException(e);
}
return i;
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}
}

21
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -1,23 +1,13 @@
package com.alibaba.otter.canal.client.adapter.hbase.service;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
import com.alibaba.otter.canal.client.adapter.hbase.support.*;
import com.alibaba.otter.canal.client.adapter.support.Dml;
/**
@@ -32,8 +22,8 @@ public class HbaseSyncService {
private HbaseTemplate hbaseTemplate; // HBase操作模板
public HbaseSyncService(Connection conn){
hbaseTemplate = new HbaseTemplate(conn);
public HbaseSyncService(HbaseTemplate hbaseTemplate){
this.hbaseTemplate = hbaseTemplate;
}
public void sync(MappingConfig config, Dml dml) {
@@ -375,7 +365,8 @@ public class HbaseSyncService {
* @param value 值
* @return 复合字段rowKey
*/
private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm, Object value) {
private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm,
Object value) {
if (value == null) {
return null;
}