chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed adapter throw exception

Browse Source
This commit is contained in:
agapple
2018-12-28 16:55:44 +08:00
parent f100aeda39
commit f72ac9a3bb
7 changed files with 204 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -100,7 +100,7 @@ public class ESSyncService {
dml.getDestination(), dml.getDestination(),
config.getEsMapping().get_index()); config.getEsMapping().get_index());
} }
} catch (Exception e) { } catch (Throwable e) {
logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml); logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
throw new RuntimeException(e); throw new RuntimeException(e);
} }

10
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java
View File

@@ -9,7 +9,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -113,6 +118,7 @@ public class HbaseTemplate {
flag = true; flag = true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e);
} }
return flag; return flag;
@@ -145,6 +151,7 @@ public class HbaseTemplate {
flag = true; flag = true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e);
} }
return flag; return flag;
} }
@@ -171,6 +178,7 @@ public class HbaseTemplate {
flag = true; flag = true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e);
} }
return flag; return flag;
} }

9
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,7 +2,10 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -127,8 +130,8 @@ public abstract class AbstractCanalAdapterWorker {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage, CanalMQConnector connector, protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
ExecutorService workerExecutor) { CanalMQConnector connector, ExecutorService workerExecutor) {
try { try {
List<?> messages; List<?> messages;
if (!flatMessage) { if (!flatMessage) {

28
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -26,7 +26,12 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
import com.alibaba.otter.canal.client.adapter.support.*; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.client.adapter.support.Util;
/** /**
* RDB适配器实现类 * RDB适配器实现类
@@ -73,8 +78,8 @@ public class RdbAdapter implements OuterAdapter {
// 过滤不匹配的key的配置 // 过滤不匹配的key的配置
rdbMappingTmp.forEach((key, mappingConfig) -> { rdbMappingTmp.forEach((key, mappingConfig) -> {
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|| (mappingConfig.getOuterAdapterKey() != null || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
&& mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { .equalsIgnoreCase(configuration.getKey()))) {
rdbMapping.put(key, mappingConfig); rdbMapping.put(key, mappingConfig);
} }
}); });
@@ -87,17 +92,17 @@ public class RdbAdapter implements OuterAdapter {
String configName = entry.getKey(); String configName = entry.getKey();
MappingConfig mappingConfig = entry.getValue(); MappingConfig mappingConfig = entry.getValue();
if (!mappingConfig.getDbMapping().getMirrorDb()) { if (!mappingConfig.getDbMapping().getMirrorDb()) {
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent( String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping() + mappingConfig.getDbMapping().getDatabase() + "."
.getDatabase() + "." + mappingConfig.getDbMapping().getTable(), + mappingConfig.getDbMapping().getTable();
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
k1 -> new ConcurrentHashMap<>()); k1 -> new ConcurrentHashMap<>());
configMap.put(configName, mappingConfig); configMap.put(configName, mappingConfig);
} else { } else {
// mirrorDB // mirrorDB
String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping().getDatabase();
+ mappingConfig.getDbMapping().getDatabase(), mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
MirrorDbConfig.create(configName, mappingConfig));
} }
} }
@@ -110,10 +115,11 @@ public class RdbAdapter implements OuterAdapter {
dataSource.setPassword(properties.get("jdbc.password")); dataSource.setPassword(properties.get("jdbc.password"));
dataSource.setInitialSize(1); dataSource.setInitialSize(1);
dataSource.setMinIdle(1); dataSource.setMinIdle(1);
dataSource.setMaxActive(10); dataSource.setMaxActive(30);
dataSource.setMaxWait(60000); dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000); dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000); dataSource.setMinEvictableIdleTimeMillis(300000);
dataSource.setUseUnfairLock(true);
try { try {
dataSource.init(); dataSource.init();

37
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java
View File

@@ -36,8 +36,7 @@ public class RdbEtlService {
/** /**
* 导入数据 * 导入数据
*/ */
public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, List<String> params) {
List<String> params) {
EtlResult etlResult = new EtlResult(); EtlResult etlResult = new EtlResult();
AtomicLong successCount = new AtomicLong(); AtomicLong successCount = new AtomicLong();
List<String> errMsg = new ArrayList<>(); List<String> errMsg = new ArrayList<>();
@@ -54,8 +53,8 @@ public class RdbEtlService {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
// 拼接sql // 拼接sql
StringBuilder sql = new StringBuilder( StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "."
"SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable()); + dbMapping.getTable());
// 拼接条件 // 拼接条件
appendCondition(params, dbMapping, srcDS, sql); appendCondition(params, dbMapping, srcDS, sql);
@@ -92,8 +91,12 @@ public class RdbEtlService {
} else { } else {
sqlFinal = sql + " LIMIT " + offset + "," + cnt; sqlFinal = sql + " LIMIT " + offset + "," + cnt;
} }
Future<Boolean> future = executor Future<Boolean> future = executor.submit(() -> executeSqlImport(srcDS,
.submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg)); targetDS,
sqlFinal,
dbMapping,
successCount,
errMsg));
futures.add(future); futures.add(future);
} }
@@ -106,10 +109,11 @@ public class RdbEtlService {
executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg); executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg);
} }
logger.info( logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000
dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); + "s!");
etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条"); etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get()
+ " 条");
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
@@ -123,8 +127,8 @@ public class RdbEtlService {
return etlResult; return etlResult;
} }
private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds, private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds, StringBuilder sql)
StringBuilder sql) throws SQLException { throws SQLException {
if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) { if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
AtomicBoolean stExists = new AtomicBoolean(false); AtomicBoolean stExists = new AtomicBoolean(false);
// 验证是否有SYS_TIME字段 // 验证是否有SYS_TIME字段
@@ -188,8 +192,7 @@ public class RdbEtlService {
StringBuilder insertSql = new StringBuilder(); StringBuilder insertSql = new StringBuilder();
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
columnsMap columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
int len = insertSql.length(); int len = insertSql.length();
insertSql.delete(len - 1, len).append(") VALUES ("); insertSql.delete(len - 1, len).append(") VALUES (");
@@ -208,8 +211,8 @@ public class RdbEtlService {
// 删除数据 // 删除数据
Map<String, Object> values = new LinkedHashMap<>(); Map<String, Object> values = new LinkedHashMap<>();
StringBuilder deleteSql = new StringBuilder( StringBuilder deleteSql = new StringBuilder("DELETE FROM " + SyncUtil.getDbTableName(dbMapping)
"DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE "); + " WHERE ");
appendCondition(dbMapping, deleteSql, values, rs); appendCondition(dbMapping, deleteSql, values, rs);
try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) { try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
int k = 1; int k = 1;
@@ -275,8 +278,8 @@ public class RdbEtlService {
/** /**
* 拼接目标表主键where条件 * 拼接目标表主键where条件
*/ */
private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values, private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values, ResultSet rs)
ResultSet rs) throws SQLException { throws SQLException {
// 拼接主键 // 拼接主键
for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) { for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
String targetColumnName = entry.getKey(); String targetColumnName = entry.getKey();

8
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java
View File

@@ -70,8 +70,10 @@ public class RdbMirrorDbSyncService {
} }
} }
if (!dmlList.isEmpty()) { if (!dmlList.isEmpty()) {
rdbSyncService.sync(dmlList, dml -> { rdbSyncService.sync(dmlList,
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase()); dml -> {
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "."
+ dml.getDatabase());
if (mirrorDbConfig == null) { if (mirrorDbConfig == null) {
return false; return false;
} }
@@ -148,7 +150,7 @@ public class RdbMirrorDbSyncService {
logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase()); logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); throw new RuntimeException(e);
} }
} }
} }

15
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -7,7 +7,11 @@ import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.*; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Function; import java.util.function.Function;
import javax.sql.DataSource; import javax.sql.DataSource;
@@ -74,7 +78,7 @@ public class RdbSyncService {
executorThreads[i] = Executors.newSingleThreadExecutor(); executorThreads[i] = Executors.newSingleThreadExecutor();
} }
} catch (SQLException e) { } catch (SQLException e) {
logger.error(e.getMessage(), e); throw new RuntimeException(e);
} }
} }
@@ -99,12 +103,13 @@ public class RdbSyncService {
int j = i; int j = i;
futures.add(executorThreads[i].submit(() -> { futures.add(executorThreads[i].submit(() -> {
try { try {
dmlsPartition[j] dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); syncItem.config,
syncItem.singleDml));
dmlsPartition[j].clear(); dmlsPartition[j].clear();
batchExecutors[j].commit(); batchExecutors[j].commit();
return true; return true;
} catch (Exception e) { } catch (Throwable e) {
batchExecutors[j].rollback(); batchExecutors[j].rollback();
throw new RuntimeException(e); throw new RuntimeException(e);
} }