chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

canal server端 MQ 配置修改

Browse Source 
MQ配置可动态加载
This commit is contained in:
mcy
2018-11-09 16:39:52 +08:00
parent 2ac84b4320
commit c5bd42bbf1
17 changed files with 333 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
server/pom.xml
View File

@@ -26,11 +26,6 @@
<version>${project.version}</version>
</dependency>
<!-- kafka -->
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.17</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>

33
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.common;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
@@ -12,18 +10,19 @@ import java.util.Map;
*/
public class MQProperties {
private String servers = "127.0.0.1:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout = 100L;
private boolean flatMessage = true;
private String servers = "127.0.0.1:6667";
private int retries = 0;
private int batchSize = 16384;
private int lingerMs = 1;
private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50;
private Long canalGetTimeout = 100L;
private boolean flatMessage = true;
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
// private List<CanalDestination> canalDestinations = new
// ArrayList<CanalDestination>();
public static class CanalDestination {
@@ -138,14 +137,6 @@ public class MQProperties {
this.flatMessage = flatMessage;
}
public List<CanalDestination> getCanalDestinations() {
return canalDestinations;
}
public void setCanalDestinations(List<CanalDestination> canalDestinations) {
this.canalDestinations = canalDestinations;
}
public boolean isFilterTransactionEntry() {
return filterTransactionEntry;
}

98
server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java
View File

@@ -1,17 +1,15 @@
package com.alibaba.otter.canal.server;
import java.io.FileInputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalMQConfig;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
@@ -20,37 +18,28 @@ import com.alibaba.otter.canal.spi.CanalMQProducer;
public class CanalMQStarter {
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private volatile boolean running = false;
private volatile boolean running = false;
private ExecutorService executorService;
private ExecutorService executorService;
private CanalMQProducer canalMQProducer;
private CanalMQProducer canalMQProducer;
private MQProperties properties;
private MQProperties properties;
private CanalServerWithEmbedded canalServer;
public CanalMQStarter(CanalMQProducer canalMQProducer){
this.canalMQProducer = canalMQProducer;
}
public void init() {
public synchronized void start(MQProperties properties) {
try {
logger.info("## load MQ configurations");
String conf = System.getProperty("mq.conf", "classpath:mq.yml");
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
properties = new Yaml().loadAs(CanalMQStarter.class.getClassLoader().getResourceAsStream(conf),
MQProperties.class);
} else {
properties = new Yaml().loadAs(new FileInputStream(conf), MQProperties.class);
if (running) {
return;
}
// 初始化 kafka producer
// canalMQProducer = new CanalKafkaProducer();
this.properties = properties;
canalMQProducer.init(properties);
// set filterTransactionEntry
if (properties.isFilterTransactionEntry()) {
@@ -62,36 +51,34 @@ public class CanalMQStarter {
System.setProperty("canal.instance.memory.rawEntry", "false");
}
canalServer = CanalServerWithEmbedded.instance();
// 对应每个instance启动一个worker线程
List<MQProperties.CanalDestination> destinations = properties.getCanalDestinations();
executorService = Executors.newFixedThreadPool(destinations.size());
executorService = Executors.newFixedThreadPool(canalServer.getCanalInstances().size());
logger.info("## start the MQ workers.");
for (final MQProperties.CanalDestination destination : destinations) {
for (final CanalInstance canalInstance : canalServer.getCanalInstances().values()) {
executorService.execute(new Runnable() {
@Override
public void run() {
MQProperties.CanalDestination destination = new MQProperties.CanalDestination();
destination.setCanalDestination(canalInstance.getDestination());
CanalMQConfig mqConfig = canalInstance.getMqConfig();
destination.setTopic(mqConfig.getTopic());
destination.setPartition(mqConfig.getPartition());
destination.setPartitionsNum(mqConfig.getPartitionsNum());
destination.setPartitionHash(mqConfig.getPartitionHashProperties());
worker(destination);
}
});
}
running = true;
logger.info("## the MQ workers is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the MQ workers");
running = false;
executorService.shutdown();
canalMQProducer.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping MQ workers:", e);
} finally {
logger.info("## canal MQ is down.");
}
stop();
}
});
@@ -102,15 +89,31 @@ public class CanalMQStarter {
}
}
public synchronized void stop() {
if (!running) {
return;
}
try {
logger.info("## stop the MQ workers");
running = false;
executorService.shutdown();
canalMQProducer.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping MQ workers:", e);
} finally {
logger.info("## canal MQ is down.");
}
}
private void worker(MQProperties.CanalDestination destination) {
while (!running)
;
logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
final CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
logger.info("## start the MQ producer: {}.", destination.getCanalDestination());
final ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
while (running) {
try {
if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
if (!canalServer.getCanalInstances().containsKey(clientIdentity.getDestination())) {
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
@@ -118,17 +121,18 @@ public class CanalMQStarter {
}
continue;
}
server.subscribe(clientIdentity);
logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination());
canalServer.subscribe(clientIdentity);
logger.info("## the MQ producer: {} is running now ......", destination.getCanalDestination());
Long getTimeout = properties.getCanalGetTimeout();
int getBatchSize = properties.getCanalBatchSize();
while (running) {
Message message;
if (getTimeout != null && getTimeout > 0) {
message = server.getWithoutAck(clientIdentity, getBatchSize, getTimeout, TimeUnit.MILLISECONDS);
message = canalServer
.getWithoutAck(clientIdentity, getBatchSize, getTimeout, TimeUnit.MILLISECONDS);
} else {
message = server.getWithoutAck(clientIdentity, getBatchSize);
message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
}
final long batchId = message.getId();
@@ -139,12 +143,12 @@ public class CanalMQStarter {
@Override
public void commit() {
server.ack(clientIdentity, batchId); // 提交确认
canalServer.ack(clientIdentity, batchId); // 提交确认
}
@Override
public void rollback() {
server.rollback(clientIdentity, batchId);
canalServer.rollback(clientIdentity, batchId);
}
}); // 发送message到topic
} else {