mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed MR
This commit is contained in:
@@ -132,6 +132,12 @@
|
|||||||
<version>1.1.1</version>
|
<version>1.1.1</version>
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<!-- Pulsar -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.pulsar</groupId>
|
||||||
|
<artifactId>pulsar-client</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<!-- junit -->
|
<!-- junit -->
|
||||||
<dependency>
|
<dependency>
|
||||||
@@ -139,12 +145,6 @@
|
|||||||
<artifactId>junit</artifactId>
|
<artifactId>junit</artifactId>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- Pulsar -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.pulsar</groupId>
|
|
||||||
<artifactId>pulsar-client</artifactId>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
|||||||
@@ -92,12 +92,12 @@
|
|||||||
</repositories>
|
</repositories>
|
||||||
|
|
||||||
<modules>
|
<modules>
|
||||||
<module>kafka-connector</module>
|
|
||||||
<module>core</module>
|
<module>core</module>
|
||||||
|
<module>kafka-connector</module>
|
||||||
<module>rocketmq-connector</module>
|
<module>rocketmq-connector</module>
|
||||||
<module>rabbitmq-connector</module>
|
<module>rabbitmq-connector</module>
|
||||||
<module>tcp-connector</module>
|
|
||||||
<module>pulsarmq-connector</module>
|
<module>pulsarmq-connector</module>
|
||||||
|
<module>tcp-connector</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
|||||||
@@ -46,8 +46,7 @@ import java.util.stream.Collectors;
|
|||||||
* @author rewerma 2020-01-27
|
* @author rewerma 2020-01-27
|
||||||
* @version 1.0.0
|
* @version 1.0.0
|
||||||
*/
|
*/
|
||||||
@SPI("rocketmq")
|
@SPI("rocketmq") public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
|
||||||
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
|
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
|
||||||
|
|
||||||
@@ -55,8 +54,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
|
|||||||
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
|
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
|
||||||
protected ThreadPoolExecutor sendPartitionExecutor;
|
protected ThreadPoolExecutor sendPartitionExecutor;
|
||||||
|
|
||||||
@Override
|
@Override public void init(Properties properties) {
|
||||||
public void init(Properties properties) {
|
|
||||||
RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
|
RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
|
||||||
this.mqProperties = rocketMQProperties;
|
this.mqProperties = rocketMQProperties;
|
||||||
super.init(properties);
|
super.init(properties);
|
||||||
@@ -145,8 +143,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message,
|
||||||
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
|
Callback callback) {
|
||||||
ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
|
ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
|
||||||
try {
|
try {
|
||||||
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
|
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
|
||||||
@@ -181,13 +179,14 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
|
public void send(final MQDestination destination, String topicName,
|
||||||
|
com.alibaba.otter.canal.protocol.Message message) {
|
||||||
// 获取当前topic的分区数
|
// 获取当前topic的分区数
|
||||||
Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
|
Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
|
||||||
destination.getDynamicTopicPartitionNum());
|
destination.getDynamicTopicPartitionNum());
|
||||||
|
|
||||||
// 获取topic的队列数为分区数
|
// 获取topic的队列数为分区数
|
||||||
if(partitionNum == null){
|
if (partitionNum == null) {
|
||||||
partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
|
partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -305,8 +304,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("deprecation")
|
@SuppressWarnings("deprecation") private void sendMessage(List<Message> messages, int partition) {
|
||||||
private void sendMessage(List<Message> messages, int partition) {
|
|
||||||
if (messages.isEmpty()) {
|
if (messages.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -348,8 +346,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override public void stop() {
|
||||||
public void stop() {
|
|
||||||
logger.info("## Stop RocketMQ producer##");
|
logger.info("## Stop RocketMQ producer##");
|
||||||
this.defaultMQProducer.shutdown();
|
this.defaultMQProducer.shutdown();
|
||||||
if (sendPartitionExecutor != null) {
|
if (sendPartitionExecutor != null) {
|
||||||
@@ -359,14 +356,14 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
|
|||||||
super.stop();
|
super.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Integer getTopicDynamicQueuesSize(Boolean enable, String topicName){
|
private Integer getTopicDynamicQueuesSize(Boolean enable, String topicName) {
|
||||||
if(enable!=null && enable){
|
if (enable != null && enable) {
|
||||||
topicName = this.defaultMQProducer.withNamespace(topicName);
|
topicName = this.defaultMQProducer.withNamespace(topicName);
|
||||||
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
|
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
|
||||||
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topicName);
|
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topicName);
|
||||||
if(topicInfo == null){
|
if (topicInfo == null) {
|
||||||
return null;
|
return null;
|
||||||
}else{
|
} else {
|
||||||
return topicInfo.getMessageQueueList().size();
|
return topicInfo.getMessageQueueList().size();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -223,7 +223,7 @@ public class BinlogDownloadQueue {
|
|||||||
TarArchiveEntry tarArchiveEntry = null;
|
TarArchiveEntry tarArchiveEntry = null;
|
||||||
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
|
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
|
||||||
String name = tarArchiveEntry.getName();
|
String name = tarArchiveEntry.getName();
|
||||||
File tarFile = new File(parentFile, name);
|
File tarFile = new File(parentFile, name + ".tmp");
|
||||||
logger.info("start to download file " + tarFile.getName());
|
logger.info("start to download file " + tarFile.getName());
|
||||||
if (tarFile.exists()) {
|
if (tarFile.exists()) {
|
||||||
tarFile.delete();
|
tarFile.delete();
|
||||||
@@ -244,7 +244,7 @@ public class BinlogDownloadQueue {
|
|||||||
}
|
}
|
||||||
tais.close();
|
tais.close();
|
||||||
} else {
|
} else {
|
||||||
File file = new File(parentFile, fileName);
|
File file = new File(parentFile, fileName + ".tmp");
|
||||||
if (file.exists()) {
|
if (file.exists()) {
|
||||||
file.delete();
|
file.delete();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user