chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed adapter

Browse Source
This commit is contained in:
agapple
2018-11-21 22:49:22 +08:00
parent cc97f3b331
commit aa89453bf6
5 changed files with 53 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java
View File

@@ -37,8 +37,6 @@ public class AdapterCanalConfig extends CanalClientConfig {
for (CanalAdapter canalAdapter : canalAdapters) {
if (canalAdapter.getInstance() != null) {
DESTINATIONS.add(canalAdapter.getInstance());
} else if (canalAdapter.getTopic() != null) {
DESTINATIONS.add(canalAdapter.getInstance());
}
}
}

32
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,9 +2,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -12,6 +15,7 @@ import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.CanalMQConnector;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.protocol.FlatMessage;
@@ -180,18 +184,22 @@ public abstract class AbstractCanalAdapterWorker {
*/
private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
// 分批同步
int len = 0;
List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) {
dmlsBatch.add(dml);
len += dml.getData().size();
if (len >= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmlsBatch);
dmlsBatch.clear();
len = 0;
if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmls);
} else {
int len = 0;
List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) {
dmlsBatch.add(dml);
len += dml.getData().size();
if (len >= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmlsBatch);
dmlsBatch.clear();
len = 0;
}
}
adapter.sync(dmlsBatch);
}
adapter.sync(dmlsBatch);
}
public void start() {

16
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -97,14 +97,14 @@ public class CanalAdapterLoader {
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
canalClientConfig.getMqServers(),
canalAdapter.getTopic(),
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getTopic() + "-" + group.getGroupId());
logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
+ group.getGroupId());
}
}
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
@@ -119,15 +119,15 @@ public class CanalAdapterLoader {
canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getMqServers(),
canalAdapter.getTopic(),
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups,
canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getTopic() + "-" + group.getGroupId());
logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
+ group.getGroupId());
}
}
}

5
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -28,10 +28,9 @@ canal.conf:
# username: root
# password: 121212
canalAdapters:
- instance: example
# topic: example
- instance: example # canal instance Name or mq topic name
groups:
- #groupId: g1
- groupId: g1
outerAdapters:
- name: logger
# - name: rdb