chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

指定是否按主键hash并行同步, 否则按表名hash表间并行同步

Browse Source
This commit is contained in:
mcy
2018-11-15 10:39:34 +08:00
parent 915849b83a
commit 997646b2db
10 changed files with 164 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -2,10 +2,7 @@ package com.alibaba.otter.canal.client.adapter.rdb;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import javax.sql.DataSource;

46
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java
View File

@@ -18,6 +18,8 @@ public class MappingConfig {
private String outerAdapterKey; // 对应适配器的key
private Boolean concurrent; // 是否并行同步
private DbMapping dbMapping; // db映射配置
public String getDataSourceKey() {
@@ -36,6 +38,14 @@ public class MappingConfig {
this.outerAdapterKey = outerAdapterKey;
}
public Boolean getConcurrent() {
return concurrent == null ? false : concurrent;
}
public void setConcurrent(Boolean concurrent) {
this.concurrent = concurrent;
}
public DbMapping getDbMapping() {
return dbMapping;
}
@@ -66,20 +76,20 @@ public class MappingConfig {
public static class DbMapping {
private String database; // 数据库名或schema名
private String table; // 表面名
private Map<String, String> targetPk; // 目标表主键字段
private boolean mapAll = false; // 映射所有字段
private String targetTable; // 目标表名
private Map<String, String> targetColumns; // 目标表字段映射
private String database; // 数据库名或schema名
private String table; // 表面名
private Map<String, String> targetPk; // 目标表主键字段
private boolean mapAll = false; // 映射所有字段
private String targetTable; // 目标表名
private Map<String, String> targetColumns; // 目标表字段映射
private String etlCondition; // etl条件sql
private String etlCondition; // etl条件sql
private Set<String> families = new LinkedHashSet<>(); // column family列表
private int readBatch = 5000;
private int commitBatch = 5000; // etl等批量提交大小
private Set<String> families = new LinkedHashSet<>(); // column family列表
private int readBatch = 5000;
private int commitBatch = 5000; // etl等批量提交大小
// private volatile Map<String, String> allColumns; // mapAll为true,自动设置改字段
// private volatile Map<String, String> allColumns; // mapAll为true,自动设置改字段
public String getDatabase() {
return database;
@@ -161,12 +171,12 @@ public class MappingConfig {
this.commitBatch = commitBatch;
}
// public Map<String, String> getAllColumns() {
// return allColumns;
// }
//
// public void setAllColumns(Map<String, String> allColumns) {
// this.allColumns = allColumns;
// }
// public Map<String, String> getAllColumns() {
// return allColumns;
// }
//
// public void setAllColumns(Map<String, String> allColumns) {
// this.allColumns = allColumns;
// }
}
}

5
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/BatchExecutor.java
View File

@@ -78,12 +78,13 @@ public class BatchExecutor {
executor.submit(() -> {
try {
commitLock.lock();
conn.commit(); //直接提交一次
if (!condition.await(5, TimeUnit.SECONDS)) {
// 超时提交
commit();
}
} catch (InterruptedException e) {
// ignore
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
commitLock.unlock();
}

66
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -48,7 +48,7 @@ public class RdbSyncService {
public RdbSyncService(Integer commitSize, Integer threads, DataSource dataSource){
try {
if (threads != null && threads > 1 && threads < 10) {
if (threads != null && threads > 1 && threads <= 10) {
this.threads = threads;
}
batchExecutors = new BatchExecutor[this.threads];
@@ -121,13 +121,24 @@ public class RdbSyncService {
Map<String, Integer> ctype = getTargetColumnType(batchExecutors[0].getConn(), config);
String sql = insertSql.toString();
Integer tbHash = null;
// 如果不是并行同步的表
if (!config.getConcurrent()) {
// 按表名hash到一个线程
tbHash = Math.abs(Math.abs(dbMapping.getTargetTable().hashCode()) % threads);
}
for (Map<String, Object> d : data) {
int pkHash = pkHash(dbMapping, d, threads);
ThreadPoolExecutor tpe = (ThreadPoolExecutor) threadExecutors[pkHash];
int hash;
if (tbHash != null) {
hash = tbHash;
} else {
hash = pkHash(dbMapping, d, threads);
}
ThreadPoolExecutor tpe = (ThreadPoolExecutor) threadExecutors[hash];
checkQueue(tpe);
tpe.submit(() -> {
try {
BatchExecutor batchExecutor = batchExecutors[pkHash];
BatchExecutor batchExecutor = batchExecutors[hash];
List<Map<String, ?>> values = new ArrayList<>();
for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
@@ -176,15 +187,26 @@ public class RdbSyncService {
Map<String, Integer> ctype = getTargetColumnType(batchExecutors[0].getConn(), config);
Integer tbHash = null;
if (!config.getConcurrent()) {
// 按表名hash到一个线程
tbHash = Math.abs(Math.abs(dbMapping.getTargetTable().hashCode()) % threads);
}
for (Map<String, Object> o : old) {
Map<String, Object> d = data.get(idx - 1);
int pkHash = pkHash(dbMapping, d, o, threads);
ThreadPoolExecutor tpe = (ThreadPoolExecutor) threadExecutors[pkHash];
int hash;
if (tbHash != null) {
hash = tbHash;
} else {
hash = pkHash(dbMapping, d, o, threads);
}
ThreadPoolExecutor tpe = (ThreadPoolExecutor) threadExecutors[hash];
checkQueue(tpe);
tpe.submit(() -> {
try {
BatchExecutor batchExecutor = batchExecutors[pkHash];
BatchExecutor batchExecutor = batchExecutors[hash];
StringBuilder updateSql = new StringBuilder();
updateSql.append("UPDATE ").append(dbMapping.getTargetTable()).append(" SET ");
@@ -211,10 +233,11 @@ public class RdbSyncService {
// 拼接主键
appendCondition(dbMapping, updateSql, ctype, values, d, o);
if (logger.isTraceEnabled()) {
logger.trace("Update target table, sql: {}", updateSql);
}
batchExecutor.execute(updateSql.toString(), values);
if (logger.isDebugEnabled()) {
logger.debug("Execute sql: {}", updateSql);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
@@ -240,15 +263,24 @@ public class RdbSyncService {
DbMapping dbMapping = config.getDbMapping();
Map<String, Integer> ctype = getTargetColumnType(batchExecutors[0].getConn(), config);
Integer tbHash = null;
if (!config.getConcurrent()) {
// 按表名hash到一个线程
tbHash = Math.abs(Math.abs(dbMapping.getTargetTable().hashCode()) % threads);
}
for (Map<String, Object> d : data) {
int pkHash = pkHash(dbMapping, d, threads);
int hash;
if (tbHash != null) {
hash = tbHash;
} else {
hash = pkHash(dbMapping, d, threads);
}
ThreadPoolExecutor tpe = (ThreadPoolExecutor) threadExecutors[pkHash];
ThreadPoolExecutor tpe = (ThreadPoolExecutor) threadExecutors[hash];
checkQueue(tpe);
tpe.submit(() -> {
try {
BatchExecutor batchExecutor = batchExecutors[pkHash];
BatchExecutor batchExecutor = batchExecutors[hash];
StringBuilder sql = new StringBuilder();
sql.append("DELETE FROM ").append(dbMapping.getTargetTable()).append(" WHERE ");
@@ -257,10 +289,10 @@ public class RdbSyncService {
// 拼接主键
appendCondition(dbMapping, sql, ctype, values, d);
if (logger.isTraceEnabled()) {
logger.trace("Delete from target table, sql: {}", sql);
}
batchExecutor.execute(sql.toString(), values);
if (logger.isDebugEnabled()) {
logger.debug("Execute sql: {}", sql);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}

1
client-adapter/rdb/src/main/resources/rdb/mytest_user.yml
View File

@@ -1,6 +1,7 @@
dataSourceKey: defaultDS
destination: example
outerAdapterKey: oracle1
concurrent: true
dbMapping:
database: mytest
table: user

9
client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java
View File

@@ -1,20 +1,21 @@
package com.alibaba.otter.canal.client.adapter.rdb.test.sync;
import java.util.HashMap;
import java.util.Map;
import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
import com.alibaba.otter.canal.client.adapter.rdb.test.TestConstant;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import java.util.HashMap;
import java.util.Map;
public class Common {
public static RdbAdapter init() {
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
outerAdapterConfig.setName("rdb");
//outerAdapterConfig.setKey("oralce1");
outerAdapterConfig.setKey("oracle1");
Map<String, String> properties = new HashMap<>();
properties.put("jdbc.driveClassName", "oracle.jdbc.OracleDriver");
properties.put("jdbc.url", "jdbc:oracle:thin:@127.0.0.1:49161:XE");

2
client-adapter/rdb/src/test/resources/rdb/mytest_user.yml
View File

@@ -1,5 +1,7 @@
dataSourceKey: defaultDS
destination: example
outerAdapterKey: oracle1
concurrent: true
dbMapping:
database: mytest
table: user