chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rdb 批量提交超时提交

Browse Source
This commit is contained in:
mcy
2018-11-17 04:37:46 +08:00
parent 991418ce6e
commit fcdf95b5ec
3 changed files with 14 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -1,14 +1,16 @@
package com.alibaba.otter.canal.client.adapter.es; package com.alibaba.otter.canal.client.adapter.es;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.*; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.sql.DataSource; import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
@@ -22,6 +24,7 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.config.SqlParser; import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService; import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;

1
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -14,7 +14,6 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;

21
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -5,7 +5,6 @@ import java.sql.SQLException;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
@@ -47,7 +46,6 @@ public class RdbAdapter implements OuterAdapter {
private List<SimpleDml> dmlList = Collections private List<SimpleDml> dmlList = Collections
.synchronizedList(new ArrayList<>()); .synchronizedList(new ArrayList<>());
private Lock syncLock = new ReentrantLock(); private Lock syncLock = new ReentrantLock();
private Condition condition = syncLock.newCondition();
private ExecutorService executor = Executors.newFixedThreadPool(1); private ExecutorService executor = Executors.newFixedThreadPool(1);
private RdbConfigMonitor rdbConfigMonitor; private RdbConfigMonitor rdbConfigMonitor;
@@ -112,16 +110,16 @@ public class RdbAdapter implements OuterAdapter {
executor.submit(() -> { executor.submit(() -> {
while (running) { while (running) {
int beginSize = dmlList.size();
try { try {
syncLock.lock(); Thread.sleep(3000);
if (!condition.await(3, TimeUnit.SECONDS)) { } catch (InterruptedException e) {
// 超时提交 // ignore
sync(); }
} int endSize = dmlList.size();
} catch (Exception e) {
logger.error(e.getMessage(), e); if (endSize - beginSize < 300) {
} finally { sync();
syncLock.unlock();
} }
} }
}); });
@@ -156,7 +154,6 @@ public class RdbAdapter implements OuterAdapter {
try { try {
syncLock.lock(); syncLock.lock();
if (!dmlList.isEmpty()) { if (!dmlList.isEmpty()) {
condition.signal();
rdbSyncService.sync(dmlList); rdbSyncService.sync(dmlList);
dmlList.clear(); dmlList.clear();
} }