chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rdb 批量提交

Browse Source
This commit is contained in:
mcy
2018-11-14 18:21:50 +08:00
parent 827b872cf8
commit d396a0b65c
3 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -36,6 +36,7 @@ canal.conf:
# jdbc.username: mytest # jdbc.username: mytest
# jdbc.password: m121212 # jdbc.password: m121212
# threads: 5 # threads: 5
# commitSize: 3000
# - name: rdb # - name: rdb
# key: postgres1 # key: postgres1
# properties: # properties:

9
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -71,11 +71,10 @@ public class RdbAdapter implements OuterAdapter {
} }
String threads = properties.get("threads"); String threads = properties.get("threads");
if (threads != null) { String commitSize = properties.get("commitSize");
rdbSyncService = new RdbSyncService(Integer.valueOf(threads), dataSource); rdbSyncService = new RdbSyncService(commitSize != null ? Integer.valueOf(commitSize) : null,
} else { threads != null ? Integer.valueOf(threads) : null,
rdbSyncService = new RdbSyncService(null, dataSource); dataSource);
}
} }
@Override @Override

9
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -9,7 +9,10 @@ import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.*; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import javax.sql.DataSource; import javax.sql.DataSource;
@@ -43,7 +46,7 @@ public class RdbSyncService {
private ExecutorService[] threadExecutors; private ExecutorService[] threadExecutors;
public RdbSyncService(Integer threads, DataSource dataSource){ public RdbSyncService(Integer commitSize, Integer threads, DataSource dataSource){
try { try {
if (threads != null && threads > 1 && threads < 10) { if (threads != null && threads > 1 && threads < 10) {
this.threads = threads; this.threads = threads;
@@ -52,7 +55,7 @@ public class RdbSyncService {
for (int i = 0; i < this.threads; i++) { for (int i = 0; i < this.threads; i++) {
Connection conn = dataSource.getConnection(); Connection conn = dataSource.getConnection();
conn.setAutoCommit(false); conn.setAutoCommit(false);
this.batchExecutors[i] = new BatchExecutor(i, conn, null); this.batchExecutors[i] = new BatchExecutor(i, conn, commitSize);
} }
threadExecutors = new ExecutorService[this.threads]; threadExecutors = new ExecutorService[this.threads];
for (int i = 0; i < this.threads; i++) { for (int i = 0; i < this.threads; i++) {