mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed adapter throw exception
This commit is contained in:
@@ -100,7 +100,7 @@ public class ESSyncService {
|
|||||||
dml.getDestination(),
|
dml.getDestination(),
|
||||||
config.getEsMapping().get_index());
|
config.getEsMapping().get_index());
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Throwable e) {
|
||||||
logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
|
logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,12 @@ import org.apache.hadoop.conf.Configuration;
|
|||||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
import org.apache.hadoop.hbase.client.*;
|
import org.apache.hadoop.hbase.client.Connection;
|
||||||
|
import org.apache.hadoop.hbase.client.ConnectionFactory;
|
||||||
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
|
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||||
|
import org.apache.hadoop.hbase.client.HTable;
|
||||||
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
@@ -113,6 +118,7 @@ public class HbaseTemplate {
|
|||||||
flag = true;
|
flag = true;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
return flag;
|
return flag;
|
||||||
|
|
||||||
@@ -145,6 +151,7 @@ public class HbaseTemplate {
|
|||||||
flag = true;
|
flag = true;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
return flag;
|
return flag;
|
||||||
}
|
}
|
||||||
@@ -171,6 +178,7 @@ public class HbaseTemplate {
|
|||||||
flag = true;
|
flag = true;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
return flag;
|
return flag;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
|
|||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.*;
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
@@ -127,8 +130,8 @@ public abstract class AbstractCanalAdapterWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage, CanalMQConnector connector,
|
protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
|
||||||
ExecutorService workerExecutor) {
|
CanalMQConnector connector, ExecutorService workerExecutor) {
|
||||||
try {
|
try {
|
||||||
List<?> messages;
|
List<?> messages;
|
||||||
if (!flatMessage) {
|
if (!flatMessage) {
|
||||||
@@ -157,7 +160,7 @@ public abstract class AbstractCanalAdapterWorker {
|
|||||||
}
|
}
|
||||||
connector.ack();
|
connector.ack();
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
if (i == retry - 1) {
|
if (i == retry - 1) {
|
||||||
connector.ack();
|
connector.ack();
|
||||||
@@ -173,7 +176,7 @@ public abstract class AbstractCanalAdapterWorker {
|
|||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -26,7 +26,12 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
|
|||||||
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
|
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
|
||||||
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
|
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
|
||||||
import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
|
import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.*;
|
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RDB适配器实现类
|
* RDB适配器实现类
|
||||||
@@ -73,8 +78,8 @@ public class RdbAdapter implements OuterAdapter {
|
|||||||
// 过滤不匹配的key的配置
|
// 过滤不匹配的key的配置
|
||||||
rdbMappingTmp.forEach((key, mappingConfig) -> {
|
rdbMappingTmp.forEach((key, mappingConfig) -> {
|
||||||
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|
||||||
|| (mappingConfig.getOuterAdapterKey() != null
|
|| (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
|
||||||
&& mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
|
.equalsIgnoreCase(configuration.getKey()))) {
|
||||||
rdbMapping.put(key, mappingConfig);
|
rdbMapping.put(key, mappingConfig);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -87,17 +92,17 @@ public class RdbAdapter implements OuterAdapter {
|
|||||||
String configName = entry.getKey();
|
String configName = entry.getKey();
|
||||||
MappingConfig mappingConfig = entry.getValue();
|
MappingConfig mappingConfig = entry.getValue();
|
||||||
if (!mappingConfig.getDbMapping().getMirrorDb()) {
|
if (!mappingConfig.getDbMapping().getMirrorDb()) {
|
||||||
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(
|
String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
|
||||||
StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping()
|
+ mappingConfig.getDbMapping().getDatabase() + "."
|
||||||
.getDatabase() + "." + mappingConfig.getDbMapping().getTable(),
|
+ mappingConfig.getDbMapping().getTable();
|
||||||
|
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
|
||||||
k1 -> new ConcurrentHashMap<>());
|
k1 -> new ConcurrentHashMap<>());
|
||||||
configMap.put(configName, mappingConfig);
|
configMap.put(configName, mappingConfig);
|
||||||
} else {
|
} else {
|
||||||
// mirrorDB
|
// mirrorDB
|
||||||
|
String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
|
||||||
mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
|
+ mappingConfig.getDbMapping().getDatabase();
|
||||||
+ mappingConfig.getDbMapping().getDatabase(),
|
mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
|
||||||
MirrorDbConfig.create(configName, mappingConfig));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,10 +115,11 @@ public class RdbAdapter implements OuterAdapter {
|
|||||||
dataSource.setPassword(properties.get("jdbc.password"));
|
dataSource.setPassword(properties.get("jdbc.password"));
|
||||||
dataSource.setInitialSize(1);
|
dataSource.setInitialSize(1);
|
||||||
dataSource.setMinIdle(1);
|
dataSource.setMinIdle(1);
|
||||||
dataSource.setMaxActive(10);
|
dataSource.setMaxActive(30);
|
||||||
dataSource.setMaxWait(60000);
|
dataSource.setMaxWait(60000);
|
||||||
dataSource.setTimeBetweenEvictionRunsMillis(60000);
|
dataSource.setTimeBetweenEvictionRunsMillis(60000);
|
||||||
dataSource.setMinEvictableIdleTimeMillis(300000);
|
dataSource.setMinEvictableIdleTimeMillis(300000);
|
||||||
|
dataSource.setUseUnfairLock(true);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
dataSource.init();
|
dataSource.init();
|
||||||
|
|||||||
@@ -36,8 +36,7 @@ public class RdbEtlService {
|
|||||||
/**
|
/**
|
||||||
* 导入数据
|
* 导入数据
|
||||||
*/
|
*/
|
||||||
public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config,
|
public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, List<String> params) {
|
||||||
List<String> params) {
|
|
||||||
EtlResult etlResult = new EtlResult();
|
EtlResult etlResult = new EtlResult();
|
||||||
AtomicLong successCount = new AtomicLong();
|
AtomicLong successCount = new AtomicLong();
|
||||||
List<String> errMsg = new ArrayList<>();
|
List<String> errMsg = new ArrayList<>();
|
||||||
@@ -54,8 +53,8 @@ public class RdbEtlService {
|
|||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
|
||||||
// 拼接sql
|
// 拼接sql
|
||||||
StringBuilder sql = new StringBuilder(
|
StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "."
|
||||||
"SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable());
|
+ dbMapping.getTable());
|
||||||
|
|
||||||
// 拼接条件
|
// 拼接条件
|
||||||
appendCondition(params, dbMapping, srcDS, sql);
|
appendCondition(params, dbMapping, srcDS, sql);
|
||||||
@@ -92,8 +91,12 @@ public class RdbEtlService {
|
|||||||
} else {
|
} else {
|
||||||
sqlFinal = sql + " LIMIT " + offset + "," + cnt;
|
sqlFinal = sql + " LIMIT " + offset + "," + cnt;
|
||||||
}
|
}
|
||||||
Future<Boolean> future = executor
|
Future<Boolean> future = executor.submit(() -> executeSqlImport(srcDS,
|
||||||
.submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg));
|
targetDS,
|
||||||
|
sqlFinal,
|
||||||
|
dbMapping,
|
||||||
|
successCount,
|
||||||
|
errMsg));
|
||||||
futures.add(future);
|
futures.add(future);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,10 +109,11 @@ public class RdbEtlService {
|
|||||||
executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg);
|
executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(
|
logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000
|
||||||
dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
|
+ "s!");
|
||||||
|
|
||||||
etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据:" + successCount.get() + " 条");
|
etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据:" + successCount.get()
|
||||||
|
+ " 条");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
|
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
|
||||||
@@ -123,8 +127,8 @@ public class RdbEtlService {
|
|||||||
return etlResult;
|
return etlResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds,
|
private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds, StringBuilder sql)
|
||||||
StringBuilder sql) throws SQLException {
|
throws SQLException {
|
||||||
if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
|
if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
|
||||||
AtomicBoolean stExists = new AtomicBoolean(false);
|
AtomicBoolean stExists = new AtomicBoolean(false);
|
||||||
// 验证是否有SYS_TIME字段
|
// 验证是否有SYS_TIME字段
|
||||||
@@ -141,9 +145,9 @@ public class RdbEtlService {
|
|||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
});
|
} );
|
||||||
if (stExists.get()) {
|
if (stExists.get()) {
|
||||||
sql.append(" WHERE SYS_TIME >= '").append(params.get(0)).append("' ");
|
sql.append(" WHERE SYS_TIME >= '").append(params.get(0)).append("' ");
|
||||||
}
|
}
|
||||||
@@ -186,85 +190,84 @@ public class RdbEtlService {
|
|||||||
// columnsMap = dbMapping.getTargetColumns();
|
// columnsMap = dbMapping.getTargetColumns();
|
||||||
// }
|
// }
|
||||||
|
|
||||||
StringBuilder insertSql = new StringBuilder();
|
StringBuilder insertSql = new StringBuilder();
|
||||||
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
|
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
|
||||||
columnsMap
|
columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
|
||||||
.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
|
|
||||||
|
|
||||||
int len = insertSql.length();
|
int len = insertSql.length();
|
||||||
insertSql.delete(len - 1, len).append(") VALUES (");
|
insertSql.delete(len - 1, len).append(") VALUES (");
|
||||||
int mapLen = columnsMap.size();
|
int mapLen = columnsMap.size();
|
||||||
for (int i = 0; i < mapLen; i++) {
|
for (int i = 0; i < mapLen; i++) {
|
||||||
insertSql.append("?,");
|
insertSql.append("?,");
|
||||||
}
|
|
||||||
len = insertSql.length();
|
|
||||||
insertSql.delete(len - 1, len).append(")");
|
|
||||||
try (Connection connTarget = targetDS.getConnection();
|
|
||||||
PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
|
|
||||||
connTarget.setAutoCommit(false);
|
|
||||||
|
|
||||||
while (rs.next()) {
|
|
||||||
pstmt.clearParameters();
|
|
||||||
|
|
||||||
// 删除数据
|
|
||||||
Map<String, Object> values = new LinkedHashMap<>();
|
|
||||||
StringBuilder deleteSql = new StringBuilder(
|
|
||||||
"DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE ");
|
|
||||||
appendCondition(dbMapping, deleteSql, values, rs);
|
|
||||||
try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
|
|
||||||
int k = 1;
|
|
||||||
for (Object val : values.values()) {
|
|
||||||
pstmt2.setObject(k++, val);
|
|
||||||
}
|
|
||||||
pstmt2.execute();
|
|
||||||
}
|
|
||||||
|
|
||||||
int i = 1;
|
|
||||||
for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
|
|
||||||
String targetClolumnName = entry.getKey();
|
|
||||||
String srcColumnName = entry.getValue();
|
|
||||||
if (srcColumnName == null) {
|
|
||||||
srcColumnName = targetClolumnName;
|
|
||||||
}
|
|
||||||
|
|
||||||
Integer type = columnType.get(targetClolumnName.toLowerCase());
|
|
||||||
|
|
||||||
Object value = rs.getObject(srcColumnName);
|
|
||||||
if (value != null) {
|
|
||||||
SyncUtil.setPStmt(type, pstmt, value, i);
|
|
||||||
} else {
|
|
||||||
pstmt.setNull(i, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
pstmt.execute();
|
|
||||||
if (logger.isTraceEnabled()) {
|
|
||||||
logger.trace("Insert into target table, sql: {}", insertSql);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (idx % dbMapping.getCommitBatch() == 0) {
|
|
||||||
connTarget.commit();
|
|
||||||
completed = true;
|
|
||||||
}
|
|
||||||
idx++;
|
|
||||||
successCount.incrementAndGet();
|
|
||||||
if (logger.isDebugEnabled()) {
|
|
||||||
logger.debug("successful import count:" + successCount.get());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!completed) {
|
|
||||||
connTarget.commit();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
|
|
||||||
errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage());
|
|
||||||
}
|
}
|
||||||
return idx;
|
len = insertSql.length();
|
||||||
});
|
insertSql.delete(len - 1, len).append(")");
|
||||||
|
try (Connection connTarget = targetDS.getConnection();
|
||||||
|
PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
|
||||||
|
connTarget.setAutoCommit(false);
|
||||||
|
|
||||||
|
while (rs.next()) {
|
||||||
|
pstmt.clearParameters();
|
||||||
|
|
||||||
|
// 删除数据
|
||||||
|
Map<String, Object> values = new LinkedHashMap<>();
|
||||||
|
StringBuilder deleteSql = new StringBuilder("DELETE FROM " + SyncUtil.getDbTableName(dbMapping)
|
||||||
|
+ " WHERE ");
|
||||||
|
appendCondition(dbMapping, deleteSql, values, rs);
|
||||||
|
try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
|
||||||
|
int k = 1;
|
||||||
|
for (Object val : values.values()) {
|
||||||
|
pstmt2.setObject(k++, val);
|
||||||
|
}
|
||||||
|
pstmt2.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
int i = 1;
|
||||||
|
for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
|
||||||
|
String targetClolumnName = entry.getKey();
|
||||||
|
String srcColumnName = entry.getValue();
|
||||||
|
if (srcColumnName == null) {
|
||||||
|
srcColumnName = targetClolumnName;
|
||||||
|
}
|
||||||
|
|
||||||
|
Integer type = columnType.get(targetClolumnName.toLowerCase());
|
||||||
|
|
||||||
|
Object value = rs.getObject(srcColumnName);
|
||||||
|
if (value != null) {
|
||||||
|
SyncUtil.setPStmt(type, pstmt, value, i);
|
||||||
|
} else {
|
||||||
|
pstmt.setNull(i, type);
|
||||||
|
}
|
||||||
|
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
pstmt.execute();
|
||||||
|
if (logger.isTraceEnabled()) {
|
||||||
|
logger.trace("Insert into target table, sql: {}", insertSql);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (idx % dbMapping.getCommitBatch() == 0) {
|
||||||
|
connTarget.commit();
|
||||||
|
completed = true;
|
||||||
|
}
|
||||||
|
idx++;
|
||||||
|
successCount.incrementAndGet();
|
||||||
|
if (logger.isDebugEnabled()) {
|
||||||
|
logger.debug("successful import count:" + successCount.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!completed) {
|
||||||
|
connTarget.commit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
|
||||||
|
errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage());
|
||||||
|
}
|
||||||
|
return idx;
|
||||||
|
} );
|
||||||
return true;
|
return true;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
@@ -275,8 +278,8 @@ public class RdbEtlService {
|
|||||||
/**
|
/**
|
||||||
* 拼接目标表主键where条件
|
* 拼接目标表主键where条件
|
||||||
*/
|
*/
|
||||||
private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
|
private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values, ResultSet rs)
|
||||||
ResultSet rs) throws SQLException {
|
throws SQLException {
|
||||||
// 拼接主键
|
// 拼接主键
|
||||||
for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
|
for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
|
||||||
String targetColumnName = entry.getKey();
|
String targetColumnName = entry.getKey();
|
||||||
|
|||||||
@@ -70,35 +70,37 @@ public class RdbMirrorDbSyncService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!dmlList.isEmpty()) {
|
if (!dmlList.isEmpty()) {
|
||||||
rdbSyncService.sync(dmlList, dml -> {
|
rdbSyncService.sync(dmlList,
|
||||||
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase());
|
dml -> {
|
||||||
if (mirrorDbConfig == null) {
|
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "."
|
||||||
return false;
|
+ dml.getDatabase());
|
||||||
}
|
if (mirrorDbConfig == null) {
|
||||||
String table = dml.getTable();
|
return false;
|
||||||
MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
|
}
|
||||||
|
String table = dml.getTable();
|
||||||
|
MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
|
||||||
|
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.getConcurrent()) {
|
if (config.getConcurrent()) {
|
||||||
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
||||||
singleDmls.forEach(singleDml -> {
|
singleDmls.forEach(singleDml -> {
|
||||||
int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData());
|
int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData());
|
||||||
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
|
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
|
||||||
rdbSyncService.getDmlsPartition()[hash].add(syncItem);
|
rdbSyncService.getDmlsPartition()[hash].add(syncItem);
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
int hash = 0;
|
int hash = 0;
|
||||||
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
||||||
singleDmls.forEach(singleDml -> {
|
singleDmls.forEach(singleDml -> {
|
||||||
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
|
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
|
||||||
rdbSyncService.getDmlsPartition()[hash].add(syncItem);
|
rdbSyncService.getDmlsPartition()[hash].add(syncItem);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,7 +150,7 @@ public class RdbMirrorDbSyncService {
|
|||||||
logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
|
logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,11 @@ import java.util.ArrayList;
|
|||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.*;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
|
|
||||||
import javax.sql.DataSource;
|
import javax.sql.DataSource;
|
||||||
@@ -74,7 +78,7 @@ public class RdbSyncService {
|
|||||||
executorThreads[i] = Executors.newSingleThreadExecutor();
|
executorThreads[i] = Executors.newSingleThreadExecutor();
|
||||||
}
|
}
|
||||||
} catch (SQLException e) {
|
} catch (SQLException e) {
|
||||||
logger.error(e.getMessage(), e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -99,12 +103,13 @@ public class RdbSyncService {
|
|||||||
int j = i;
|
int j = i;
|
||||||
futures.add(executorThreads[i].submit(() -> {
|
futures.add(executorThreads[i].submit(() -> {
|
||||||
try {
|
try {
|
||||||
dmlsPartition[j]
|
dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
|
||||||
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
|
syncItem.config,
|
||||||
|
syncItem.singleDml));
|
||||||
dmlsPartition[j].clear();
|
dmlsPartition[j].clear();
|
||||||
batchExecutors[j].commit();
|
batchExecutors[j].commit();
|
||||||
return true;
|
return true;
|
||||||
} catch (Exception e) {
|
} catch (Throwable e) {
|
||||||
batchExecutors[j].rollback();
|
batchExecutors[j].rollback();
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
@@ -131,41 +136,41 @@ public class RdbSyncService {
|
|||||||
sync(dmls, dml -> {
|
sync(dmls, dml -> {
|
||||||
if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
|
if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
|
||||||
// DDL
|
// DDL
|
||||||
columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
|
columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
|
||||||
|
return false;
|
||||||
|
} else {
|
||||||
|
// DML
|
||||||
|
String destination = StringUtils.trimToEmpty(dml.getDestination());
|
||||||
|
String database = dml.getDatabase();
|
||||||
|
String table = dml.getTable();
|
||||||
|
Map<String, MappingConfig> configMap = mappingConfig.get(destination + "." + database + "." + table);
|
||||||
|
|
||||||
|
if (configMap == null) {
|
||||||
return false;
|
return false;
|
||||||
} else {
|
|
||||||
// DML
|
|
||||||
String destination = StringUtils.trimToEmpty(dml.getDestination());
|
|
||||||
String database = dml.getDatabase();
|
|
||||||
String table = dml.getTable();
|
|
||||||
Map<String, MappingConfig> configMap = mappingConfig.get(destination + "." + database + "." + table);
|
|
||||||
|
|
||||||
if (configMap == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean executed = false;
|
|
||||||
for (MappingConfig config : configMap.values()) {
|
|
||||||
if (config.getConcurrent()) {
|
|
||||||
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
|
||||||
singleDmls.forEach(singleDml -> {
|
|
||||||
int hash = pkHash(config.getDbMapping(), singleDml.getData());
|
|
||||||
SyncItem syncItem = new SyncItem(config, singleDml);
|
|
||||||
dmlsPartition[hash].add(syncItem);
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
int hash = 0;
|
|
||||||
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
|
||||||
singleDmls.forEach(singleDml -> {
|
|
||||||
SyncItem syncItem = new SyncItem(config, singleDml);
|
|
||||||
dmlsPartition[hash].add(syncItem);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
executed = true;
|
|
||||||
}
|
|
||||||
return executed;
|
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
boolean executed = false;
|
||||||
|
for (MappingConfig config : configMap.values()) {
|
||||||
|
if (config.getConcurrent()) {
|
||||||
|
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
||||||
|
singleDmls.forEach(singleDml -> {
|
||||||
|
int hash = pkHash(config.getDbMapping(), singleDml.getData());
|
||||||
|
SyncItem syncItem = new SyncItem(config, singleDml);
|
||||||
|
dmlsPartition[hash].add(syncItem);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
int hash = 0;
|
||||||
|
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
|
||||||
|
singleDmls.forEach(singleDml -> {
|
||||||
|
SyncItem syncItem = new SyncItem(config, singleDml);
|
||||||
|
dmlsPartition[hash].add(syncItem);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
executed = true;
|
||||||
|
}
|
||||||
|
return executed;
|
||||||
|
}
|
||||||
|
} );
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user