chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1232 from rewerma/master

Browse Source 
fixed #1229
This commit is contained in:
agapple
2018-11-30 13:45:10 +08:00
committed by GitHub
parent 5fe7ae1613 339ecdd817
commit 19e164b599
2 changed files with 28 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -50,6 +50,8 @@ public class CanalLauncher {
if (canalMQProducer != null) { if (canalMQProducer != null) {
// disable netty // disable netty
System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true"); System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
System.setProperty(CanalConstants.CANAL_DESTINATIONS,
properties.getProperty(CanalConstants.CANAL_DESTINATIONS));
} }
logger.info("## start the canal server."); logger.info("## start the canal server.");

46
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -7,6 +7,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -60,9 +61,12 @@ public class CanalMQStarter {
// 对应每个instance启动一个worker线程 // 对应每个instance启动一个worker线程
executorService = Executors.newCachedThreadPool(); executorService = Executors.newCachedThreadPool();
logger.info("## start the MQ workers."); logger.info("## start the MQ workers.");
for (final CanalInstance canalInstance : canalServer.getCanalInstances().values()) {
CanalMQRunnable canalMQRunnable = new CanalMQRunnable(canalInstance); String[] destinations = StringUtils.split(System.getProperty("canal.destinations"), ",");
canalMQWorks.put(canalInstance.getDestination(), canalMQRunnable); for (String destination : destinations) {
destination = destination.trim();
CanalMQRunnable canalMQRunnable = new CanalMQRunnable(destination);
canalMQWorks.put(destination, canalMQRunnable);
executorService.execute(canalMQRunnable); executorService.execute(canalMQRunnable);
} }
@@ -95,7 +99,7 @@ public class CanalMQStarter {
CanalInstance canalInstance = canalServer.getCanalInstances().get(destination); CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
if (canalInstance != null) { if (canalInstance != null) {
stopDestination(destination); stopDestination(destination);
CanalMQRunnable canalMQRunnable = new CanalMQRunnable(canalInstance); CanalMQRunnable canalMQRunnable = new CanalMQRunnable(destination);
canalMQWorks.put(canalInstance.getDestination(), canalMQRunnable); canalMQWorks.put(canalInstance.getDestination(), canalMQRunnable);
executorService.execute(canalMQRunnable); executorService.execute(canalMQRunnable);
logger.info("## Start the MQ work of destination:" + destination); logger.info("## Start the MQ work of destination:" + destination);
@@ -111,15 +115,16 @@ public class CanalMQStarter {
} }
} }
private void worker(MQProperties.CanalDestination destination, AtomicBoolean destinationRunning) { private void worker(String destination, AtomicBoolean destinationRunning) {
while (!running || !destinationRunning.get()) while (!running || !destinationRunning.get())
; ;
logger.info("## start the MQ producer: {}.", destination.getCanalDestination()); logger.info("## start the MQ producer: {}.", destination);
final ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, ""); final ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1001, "");
while (running && destinationRunning.get()) { while (running && destinationRunning.get()) {
try { try {
if (!canalServer.getCanalInstances().containsKey(clientIdentity.getDestination())) { CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
if (canalInstance == null) {
try { try {
Thread.sleep(3000); Thread.sleep(3000);
} catch (InterruptedException e) { } catch (InterruptedException e) {
@@ -127,8 +132,16 @@ public class CanalMQStarter {
} }
continue; continue;
} }
MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination();
canalDestination.setCanalDestination(destination);
CanalMQConfig mqConfig = canalInstance.getMqConfig();
canalDestination.setTopic(mqConfig.getTopic());
canalDestination.setPartition(mqConfig.getPartition());
canalDestination.setPartitionsNum(mqConfig.getPartitionsNum());
canalDestination.setPartitionHash(mqConfig.getPartitionHashProperties());
canalServer.subscribe(clientIdentity); canalServer.subscribe(clientIdentity);
logger.info("## the MQ producer: {} is running now ......", destination.getCanalDestination()); logger.info("## the MQ producer: {} is running now ......", destination);
Long getTimeout = properties.getCanalGetTimeout(); Long getTimeout = properties.getCanalGetTimeout();
int getBatchSize = properties.getCanalBatchSize(); int getBatchSize = properties.getCanalBatchSize();
@@ -145,7 +158,7 @@ public class CanalMQStarter {
try { try {
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
if (batchId != -1 && size != 0) { if (batchId != -1 && size != 0) {
canalMQProducer.send(destination, message, new CanalMQProducer.Callback() { canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() {
@Override @Override
public void commit() { public void commit() {
@@ -177,23 +190,16 @@ public class CanalMQStarter {
private class CanalMQRunnable implements Runnable { private class CanalMQRunnable implements Runnable {
private CanalInstance canalInstance; private String destination;
public CanalMQRunnable(CanalInstance canalInstance){ CanalMQRunnable(String destination){
this.canalInstance = canalInstance; this.destination = destination;
} }
private AtomicBoolean running = new AtomicBoolean(true); private AtomicBoolean running = new AtomicBoolean(true);
@Override @Override
public void run() { public void run() {
MQProperties.CanalDestination destination = new MQProperties.CanalDestination();
destination.setCanalDestination(canalInstance.getDestination());
CanalMQConfig mqConfig = canalInstance.getMqConfig();
destination.setTopic(mqConfig.getTopic());
destination.setPartition(mqConfig.getPartition());
destination.setPartitionsNum(mqConfig.getPartitionsNum());
destination.setPartitionHash(mqConfig.getPartitionHashProperties());
worker(destination, running); worker(destination, running);
} }