chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

code optimization

Browse Source
This commit is contained in:
mcy
2019-03-20 10:41:51 +08:00
parent 5821c1f606
commit 06bfcda275
7 changed files with 454 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java
View File

@@ -10,6 +10,9 @@ import java.time.*;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Date; import java.util.Date;
import java.util.TimeZone; import java.util.TimeZone;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
@@ -106,6 +109,40 @@ public class Util {
return column; return column;
} }
public static ThreadPoolExecutor newFixedThreadPool(int nThreads, long keepAliveTime) {
return new ThreadPoolExecutor(nThreads,
nThreads,
keepAliveTime,
TimeUnit.MILLISECONDS,
new SynchronousQueue<>(),
(r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e) {
// ignore
}
}
});
}
public static ThreadPoolExecutor newSingleThreadExecutor(long keepAliveTime) {
return new ThreadPoolExecutor(1,
1,
keepAliveTime,
TimeUnit.MILLISECONDS,
new SynchronousQueue<>(),
(r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e) {
// ignore
}
}
});
}
public final static String timeZone; // 当前时区 public final static String timeZone; // 当前时区
private static DateTimeZone dateTimeZone; private static DateTimeZone dateTimeZone;

36
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java
View File

@@ -1,15 +1,8 @@
package com.alibaba.otter.canal.client.adapter.es.service; package com.alibaba.otter.canal.client.adapter.es.service;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@@ -121,21 +114,7 @@ public class ESEtlService {
if (cnt >= 10000) { if (cnt >= 10000) {
int threadCount = 3; // 从配置读取默认为3 int threadCount = 3; // 从配置读取默认为3
long perThreadCnt = cnt / threadCount; long perThreadCnt = cnt / threadCount;
ExecutorService executor = new ThreadPoolExecutor(threadCount, ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
threadCount,
5000L,
TimeUnit.MILLISECONDS,
new SynchronousQueue<>(),
(r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e1) {
// ignore
}
}
});
List<Future<Boolean>> futures = new ArrayList<>(threadCount);
for (int i = 0; i < threadCount; i++) { for (int i = 0; i < threadCount; i++) {
long offset = i * perThreadCnt; long offset = i * perThreadCnt;
Long size = null; Long size = null;
@@ -148,16 +127,13 @@ public class ESEtlService {
} else { } else {
sqlFinal = sql + " LIMIT " + offset + "," + cnt; sqlFinal = sql + " LIMIT " + offset + "," + cnt;
} }
Future<Boolean> future = executor executor.execute(() -> executeSqlImport(dataSource, sqlFinal, mapping, impCount, errMsg));
.submit(() -> executeSqlImport(dataSource, sqlFinal, mapping, impCount, errMsg));
futures.add(future);
}
for (Future<Boolean> future : futures) {
future.get();
} }
executor.shutdown(); executor.shutdown();
while (!executor.awaitTermination(3, TimeUnit.SECONDS)) {
// ignore
}
} else { } else {
executeSqlImport(dataSource, sql, mapping, impCount, errMsg); executeSqlImport(dataSource, sql, mapping, impCount, errMsg);
} }

798
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -1,407 +1,391 @@
package com.alibaba.otter.canal.client.adapter.hbase.service; package com.alibaba.otter.canal.client.adapter.hbase.service;
import java.sql.ResultSetMetaData; import java.sql.ResultSetMetaData;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource; import javax.sql.DataSource;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.support.HRow; import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhType; import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil; import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
import com.alibaba.otter.canal.client.adapter.hbase.support.Type; import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil; import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.alibaba.otter.canal.client.adapter.support.Util; import com.alibaba.otter.canal.client.adapter.support.Util;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
/** /**
* HBase ETL 操作业务类 * HBase ETL 操作业务类
* *
* @author rewerma @ 2018-10-20 * @author rewerma @ 2018-10-20
* @version 1.0.0 * @version 1.0.0
*/ */
public class HbaseEtlService { public class HbaseEtlService {
private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class);
/** /**
* 建表 * 建表
* *
* @param hbaseTemplate * @param hbaseTemplate
* @param config * @param config
*/ */
public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) {
try { try {
// 判断hbase表是否存在，不存在则建表 // 判断hbase表是否存在，不存在则建表
MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily()); hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily());
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
/** /**
* 导入数据 * 导入数据
* *
* @param ds 数据源 * @param ds 数据源
* @param hbaseTemplate hbaseTemplate * @param hbaseTemplate hbaseTemplate
* @param config 配置 * @param config 配置
* @param params 筛选条件 * @param params 筛选条件
* @return 导入结果 * @return 导入结果
*/ */
public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config,
List<String> params) { List<String> params) {
EtlResult etlResult = new EtlResult(); EtlResult etlResult = new EtlResult();
AtomicLong successCount = new AtomicLong(); AtomicLong successCount = new AtomicLong();
List<String> errMsg = new ArrayList<>(); List<String> errMsg = new ArrayList<>();
String hbaseTable = ""; String hbaseTable = "";
try { try {
if (config == null) { if (config == null) {
logger.error("Config is null!"); logger.error("Config is null!");
etlResult.setSucceeded(false); etlResult.setSucceeded(false);
etlResult.setErrorMessage("Config is null!"); etlResult.setErrorMessage("Config is null!");
return etlResult; return etlResult;
} }
MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
hbaseTable = hbaseMapping.getHbaseTable(); hbaseTable = hbaseMapping.getHbaseTable();
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) {
logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!"); logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!");
// 如果表存在则删除 // 如果表存在则删除
if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
hbaseTemplate.disableTable(hbaseMapping.getHbaseTable()); hbaseTemplate.disableTable(hbaseMapping.getHbaseTable());
hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable()); hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable());
} }
params = null; params = null;
} else { } else {
logger.info(hbaseMapping.getHbaseTable() + " etl is starting!"); logger.info(hbaseMapping.getHbaseTable() + " etl is starting!");
} }
createTable(hbaseTemplate, config); createTable(hbaseTemplate, config);
// 拼接sql // 拼接sql
String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable(); String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable();
// 拼接条件 // 拼接条件
if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) { if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) {
AtomicBoolean stExists = new AtomicBoolean(false); AtomicBoolean stExists = new AtomicBoolean(false);
// 验证是否有SYS_TIME字段 // 验证是否有SYS_TIME字段
Util.sqlRS(ds, sql, rs -> { Util.sqlRS(ds, sql, rs -> {
try { try {
ResultSetMetaData rsmd = rs.getMetaData(); ResultSetMetaData rsmd = rs.getMetaData();
int cnt = rsmd.getColumnCount(); int cnt = rsmd.getColumnCount();
for (int i = 1; i <= cnt; i++) { for (int i = 1; i <= cnt; i++) {
String columnName = rsmd.getColumnName(i); String columnName = rsmd.getColumnName(i);
if ("SYS_TIME".equalsIgnoreCase(columnName)) { if ("SYS_TIME".equalsIgnoreCase(columnName)) {
stExists.set(true); stExists.set(true);
break; break;
} }
} }
} catch (Exception e) { } catch (Exception e) {
// ignore // ignore
} }
return null; return null;
}); });
if (stExists.get()) { if (stExists.get()) {
sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; sql += " WHERE SYS_TIME >= '" + params.get(0) + "' ";
} }
} else if (hbaseMapping.getEtlCondition() != null && params != null) { } else if (hbaseMapping.getEtlCondition() != null && params != null) {
String etlCondition = hbaseMapping.getEtlCondition(); String etlCondition = hbaseMapping.getEtlCondition();
int size = params.size(); int size = params.size();
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
} }
sql += " " + etlCondition; sql += " " + etlCondition;
} }
// 获取总数 // 获取总数
String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
long cnt = (Long) Util.sqlRS(ds, countSql, rs -> { long cnt = (Long) Util.sqlRS(ds, countSql, rs -> {
Long count = null; Long count = null;
try { try {
if (rs.next()) { if (rs.next()) {
count = ((Number) rs.getObject(1)).longValue(); count = ((Number) rs.getObject(1)).longValue();
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
return count == null ? 0 : count; return count == null ? 0 : count;
}); });
// 当大于1万条记录时开启多线程 // 当大于1万条记录时开启多线程
if (cnt >= 10000) { if (cnt >= 10000) {
int threadCount = 3; int threadCount = 3;
long perThreadCnt = cnt / threadCount; long perThreadCnt = cnt / threadCount;
ExecutorService executor = new ThreadPoolExecutor(threadCount, ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
threadCount, for (int i = 0; i < threadCount; i++) {
5000L, long offset = i * perThreadCnt;
TimeUnit.MILLISECONDS, Long size = null;
new SynchronousQueue<>(), if (i != threadCount - 1) {
(r, exe) -> { size = perThreadCnt;
if (!exe.isShutdown()) { }
try { String sqlFinal;
exe.getQueue().put(r); if (size != null) {
} catch (InterruptedException e1) { sqlFinal = sql + " LIMIT " + offset + "," + size;
// ignore } else {
} sqlFinal = sql + " LIMIT " + offset + "," + cnt;
} }
}); executor.submit(
List<Future<Boolean>> futures = new ArrayList<>(threadCount); () -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg));
for (int i = 0; i < threadCount; i++) { }
long offset = i * perThreadCnt;
Long size = null; executor.shutdown();
if (i != threadCount - 1) { while (!executor.awaitTermination(3, TimeUnit.SECONDS)) {
size = perThreadCnt; // ignore
} }
String sqlFinal; } else {
if (size != null) { executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg);
sqlFinal = sql + " LIMIT " + offset + "," + size; }
} else {
sqlFinal = sql + " LIMIT " + offset + "," + cnt; logger.info(hbaseMapping.getHbaseTable() + " etl completed in: "
} + (System.currentTimeMillis() - start) / 1000 + "s!");
Future<Boolean> future = executor.submit(
() -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg)); etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条");
futures.add(future); } catch (Exception e) {
} logger.error(e.getMessage(), e);
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
for (Future<Boolean> future : futures) { }
future.get();
} if (errMsg.isEmpty()) {
etlResult.setSucceeded(true);
executor.shutdown(); } else {
} else { etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg); }
} return etlResult;
}
logger.info(hbaseMapping.getHbaseTable() + " etl completed in: "
+ (System.currentTimeMillis() - start) / 1000 + "s!"); /**
* 执行导入
etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条"); *
} catch (Exception e) { * @param ds
logger.error(e.getMessage(), e); * @param sql
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); * @param hbaseMapping
} * @param hbaseTemplate
* @param successCount
if (errMsg.isEmpty()) { * @param errMsg
etlResult.setSucceeded(true); * @return
} else { */
etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping,
} HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) {
return etlResult; try {
} Util.sqlRS(ds, sql, rs -> {
int i = 1;
/**
* 执行导入 try {
* boolean complete = false;
* @param ds List<HRow> rows = new ArrayList<>();
* @param sql String[] rowKeyColumns = null;
* @param hbaseMapping if (hbaseMapping.getRowKey() != null) {
* @param hbaseTemplate rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
* @param successCount }
* @param errMsg while (rs.next()) {
* @return int cc = rs.getMetaData().getColumnCount();
*/ int[] jdbcTypes = new int[cc];
private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping, Class<?>[] classes = new Class[cc];
HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) { for (int j = 1; j <= cc; j++) {
try { int jdbcType = rs.getMetaData().getColumnType(j);
Util.sqlRS(ds, sql, rs -> { jdbcTypes[j - 1] = jdbcType;
int i = 1; classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType);
}
try { HRow row = new HRow();
boolean complete = false;
List<HRow> rows = new ArrayList<>(); if (rowKeyColumns != null) {
String[] rowKeyColumns = null; // 取rowKey字段拼接
if (hbaseMapping.getRowKey() != null) { StringBuilder rowKeyVale = new StringBuilder();
rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); for (String rowKeyColumnName : rowKeyColumns) {
} Object obj = rs.getObject(rowKeyColumnName);
while (rs.next()) { if (obj != null) {
int cc = rs.getMetaData().getColumnCount(); rowKeyVale.append(obj.toString());
int[] jdbcTypes = new int[cc]; }
Class<?>[] classes = new Class[cc]; rowKeyVale.append("|");
for (int j = 1; j <= cc; j++) { }
int jdbcType = rs.getMetaData().getColumnType(j); int len = rowKeyVale.length();
jdbcTypes[j - 1] = jdbcType; if (len > 0) {
classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType); rowKeyVale.delete(len - 1, len);
} }
HRow row = new HRow(); row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
}
if (rowKeyColumns != null) {
// 取rowKey字段拼接 for (int j = 1; j <= cc; j++) {
StringBuilder rowKeyVale = new StringBuilder(); String columnName = rs.getMetaData().getColumnName(j);
for (String rowKeyColumnName : rowKeyColumns) {
Object obj = rs.getObject(rowKeyColumnName); Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
if (obj != null) { if (val == null) {
rowKeyVale.append(obj.toString()); continue;
} }
rowKeyVale.append("|");
} MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
int len = rowKeyVale.length(); // 没有配置映射
if (len > 0) { if (columnItem == null) {
rowKeyVale.delete(len - 1, len); String family = hbaseMapping.getFamily();
} String qualifile = columnName;
row.setRowKey(Bytes.toBytes(rowKeyVale.toString())); if (hbaseMapping.isUppercaseQualifier()) {
} qualifile = qualifile.toUpperCase();
}
for (int j = 1; j <= cc; j++) { if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
String columnName = rs.getMetaData().getColumnName(j); if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(Bytes.toBytes(val.toString()));
Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]); } else {
if (val == null) { row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
continue; }
} } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
Type type = Type.getType(classes[j - 1]);
MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName); if (hbaseMapping.getRowKey() == null && j == 1) {
// 没有配置映射 row.setRowKey(TypeUtil.toBytes(val, type));
if (columnItem == null) { } else {
String family = hbaseMapping.getFamily(); row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
String qualifile = columnName; }
if (hbaseMapping.isUppercaseQualifier()) { } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
qualifile = qualifile.toUpperCase(); PhType phType = PhType.getType(classes[j - 1]);
} if (hbaseMapping.getRowKey() == null && j == 1) {
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { row.setRowKey(PhTypeUtil.toBytes(val, phType));
if (hbaseMapping.getRowKey() == null && j == 1) { } else {
row.setRowKey(Bytes.toBytes(val.toString())); row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
} else { }
row.addCell(family, qualifile, Bytes.toBytes(val.toString())); }
} } else {
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { // 如果不需要类型转换
Type type = Type.getType(classes[j - 1]); if (columnItem.getType() == null || "".equals(columnItem.getType())) {
if (hbaseMapping.getRowKey() == null && j == 1) { if (val instanceof java.sql.Date) {
row.setRowKey(TypeUtil.toBytes(val, type)); SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
} else { val = dateFmt.format((Date) val);
row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); } else if (val instanceof Timestamp) {
} SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { val = datetimeFmt.format((Date) val);
PhType phType = PhType.getType(classes[j - 1]); }
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(PhTypeUtil.toBytes(val, phType)); byte[] valBytes = Bytes.toBytes(val.toString());
} else { if (columnItem.isRowKey()) {
row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); if (columnItem.getRowKeyLen() != null) {
} valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
} row.setRowKey(valBytes);
} else { } else {
// 如果不需要类型转换 row.setRowKey(valBytes);
if (columnItem.getType() == null || "".equals(columnItem.getType())) { }
if (val instanceof java.sql.Date) { } else {
SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd"); row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
val = dateFmt.format((Date) val); }
} else if (val instanceof Timestamp) { } else {
SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
val = datetimeFmt.format((Date) val); byte[] valBytes = Bytes.toBytes(val.toString());
} if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
byte[] valBytes = Bytes.toBytes(val.toString()); valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
if (columnItem.isRowKey()) { }
if (columnItem.getRowKeyLen() != null) { row.setRowKey(valBytes);
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); } else {
row.setRowKey(valBytes); row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
} else { }
row.setRowKey(valBytes); } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
} Type type = Type.getType(columnItem.getType());
} else { if (columnItem.isRowKey()) {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); if (columnItem.getRowKeyLen() != null) {
} String v = limitLenNum(columnItem.getRowKeyLen(), val);
} else { row.setRowKey(Bytes.toBytes(v));
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { } else {
byte[] valBytes = Bytes.toBytes(val.toString()); row.setRowKey(TypeUtil.toBytes(val, type));
if (columnItem.isRowKey()) { }
if (columnItem.getRowKeyLen() != null) { } else {
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); row.addCell(columnItem.getFamily(),
} columnItem.getQualifier(),
row.setRowKey(valBytes); TypeUtil.toBytes(val, type));
} else { }
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
} PhType phType = PhType.getType(columnItem.getType());
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { if (columnItem.isRowKey()) {
Type type = Type.getType(columnItem.getType()); row.setRowKey(PhTypeUtil.toBytes(val, phType));
if (columnItem.isRowKey()) { } else {
if (columnItem.getRowKeyLen() != null) { row.addCell(columnItem.getFamily(),
String v = limitLenNum(columnItem.getRowKeyLen(), val); columnItem.getQualifier(),
row.setRowKey(Bytes.toBytes(v)); PhTypeUtil.toBytes(val, phType));
} else { }
row.setRowKey(TypeUtil.toBytes(val, type)); }
} }
} else { }
row.addCell(columnItem.getFamily(), }
columnItem.getQualifier(),
TypeUtil.toBytes(val, type)); if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
}
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { rows.add(row);
PhType phType = PhType.getType(columnItem.getType()); complete = false;
if (columnItem.isRowKey()) { if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
row.setRowKey(PhTypeUtil.toBytes(val, phType)); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
} else { rows.clear();
row.addCell(columnItem.getFamily(), complete = true;
columnItem.getQualifier(), }
PhTypeUtil.toBytes(val, phType)); i++;
} successCount.incrementAndGet();
} if (logger.isDebugEnabled()) {
} logger.debug("successful import count:" + successCount.get());
} }
} }
if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空"); if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rows.add(row); }
complete = false;
if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) { } catch (Exception e) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
rows.clear(); errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
complete = true; // throw new RuntimeException(e);
} }
i++; return i;
successCount.incrementAndGet(); });
if (logger.isDebugEnabled()) { return true;
logger.debug("successful import count:" + successCount.get()); } catch (Exception e) {
} logger.error(e.getMessage(), e);
} return false;
}
if (!complete && !rows.isEmpty()) { }
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
} private static String limitLenNum(int len, Object val) {
if (val == null) {
} catch (Exception e) { return null;
logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); }
errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage()); if (val instanceof Number) {
// throw new RuntimeException(e); return String.format("%0" + len + "d", (Number) ((Number) val).longValue());
} } else if (val instanceof String) {
return i; return String.format("%0" + len + "d", Long.parseLong((String) val));
}); }
return true; return null;
} catch (Exception e) { }
logger.error(e.getMessage(), e); }
return false;
}
}
private static String limitLenNum(int len, Object val) {
if (val == null) {
return null;
}
if (val instanceof Number) {
return String.format("%0" + len + "d", (Number) ((Number) val).longValue());
} else if (val instanceof String) {
return String.format("%0" + len + "d", Long.parseLong((String) val));
}
return null;
}
}

21
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,7 +2,9 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -14,6 +16,7 @@ import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil; import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
@@ -40,21 +43,7 @@ public abstract class AbstractCanalAdapterWorker {
public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){ public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){
this.canalOuterAdapters = canalOuterAdapters; this.canalOuterAdapters = canalOuterAdapters;
int adaptersSize = canalOuterAdapters.size(); this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
this.groupInnerExecutorService = new ThreadPoolExecutor(adaptersSize,
adaptersSize,
5000L,
TimeUnit.MILLISECONDS,
new SynchronousQueue<>(),
(r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e1) {
// ignore
}
}
});
syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class); syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
} }

16
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -1,12 +1,14 @@
package com.alibaba.otter.canal.adapter.launcher.loader; package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector; import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
/** /**
@@ -47,17 +49,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
// ignore // ignore
} }
} }
ExecutorService workerExecutor = new ThreadPoolExecutor(1, 1, ExecutorService workerExecutor = Util.newSingleThreadExecutor(5000L);
5000L, TimeUnit.MILLISECONDS,
new SynchronousQueue<>(), (r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e1) {
//ignore
}
}
});
int retry = canalClientConfig.getRetries() == null int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒

18
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -1,12 +1,13 @@
package com.alibaba.otter.canal.adapter.launcher.loader; package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.ExecutorService;
import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
/** /**
@@ -48,20 +49,7 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
} }
} }
ExecutorService workerExecutor = new ThreadPoolExecutor(1, ExecutorService workerExecutor = Util.newSingleThreadExecutor(5000L);
1,
5000L,
TimeUnit.MILLISECONDS,
new SynchronousQueue<>(),
(r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e1) {
// ignore
}
}
});
int retry = canalClientConfig.getRetries() == null int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒

31
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java
View File

@@ -5,7 +5,8 @@ import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.*; import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
@@ -76,21 +77,7 @@ public class RdbEtlService {
if (cnt >= 10000) { if (cnt >= 10000) {
int threadCount = 3; int threadCount = 3;
long perThreadCnt = cnt / threadCount; long perThreadCnt = cnt / threadCount;
ExecutorService executor = new ThreadPoolExecutor(threadCount, ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
threadCount,
5000L,
TimeUnit.MILLISECONDS,
new SynchronousQueue<>(),
(r, exe) -> {
if (!exe.isShutdown()) {
try {
exe.getQueue().put(r);
} catch (InterruptedException e1) {
// ignore
}
}
});
List<Future<Boolean>> futures = new ArrayList<>(threadCount);
for (int i = 0; i < threadCount; i++) { for (int i = 0; i < threadCount; i++) {
long offset = i * perThreadCnt; long offset = i * perThreadCnt;
Long size = null; Long size = null;
@@ -103,16 +90,14 @@ public class RdbEtlService {
} else { } else {
sqlFinal = sql + " LIMIT " + offset + "," + cnt; sqlFinal = sql + " LIMIT " + offset + "," + cnt;
} }
Future<Boolean> future = executor executor
.submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg)); .execute(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg));
futures.add(future);
}
for (Future<Boolean> future : futures) {
future.get();
} }
executor.shutdown(); executor.shutdown();
while (!executor.awaitTermination(3, TimeUnit.SECONDS)) {
// ignore
}
} else { } else {
executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg); executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg);
} }