This commit is contained in:
jianghang.loujh
2021-10-09 15:59:12 +08:00
parent 5b6fd8094a
commit 40d840375f
4 changed files with 66 additions and 69 deletions

View File

@@ -132,6 +132,12 @@
<version>1.1.1</version> <version>1.1.1</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<!-- Pulsar -->
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client</artifactId>
<scope>provided</scope>
</dependency>
<!-- junit --> <!-- junit -->
<dependency> <dependency>
@@ -139,12 +145,6 @@
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- Pulsar -->
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>

View File

@@ -92,12 +92,12 @@
</repositories> </repositories>
<modules> <modules>
<module>kafka-connector</module>
<module>core</module> <module>core</module>
<module>kafka-connector</module>
<module>rocketmq-connector</module> <module>rocketmq-connector</module>
<module>rabbitmq-connector</module> <module>rabbitmq-connector</module>
<module>tcp-connector</module>
<module>pulsarmq-connector</module> <module>pulsarmq-connector</module>
<module>tcp-connector</module>
</modules> </modules>
<build> <build>

View File

@@ -46,8 +46,7 @@ import java.util.stream.Collectors;
* @author rewerma 2020-01-27 * @author rewerma 2020-01-27
* @version 1.0.0 * @version 1.0.0
*/ */
@SPI("rocketmq") @SPI("rocketmq") public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class); private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
@@ -55,8 +54,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
private static final String CLOUD_ACCESS_CHANNEL = "cloud"; private static final String CLOUD_ACCESS_CHANNEL = "cloud";
protected ThreadPoolExecutor sendPartitionExecutor; protected ThreadPoolExecutor sendPartitionExecutor;
@Override @Override public void init(Properties properties) {
public void init(Properties properties) {
RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig(); RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
this.mqProperties = rocketMQProperties; this.mqProperties = rocketMQProperties;
super.init(properties); super.init(properties);
@@ -145,8 +143,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
} }
} }
@Override @Override public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message,
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) { Callback callback) {
ExecutorTemplate template = new ExecutorTemplate(sendExecutor); ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
try { try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) { if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
@@ -181,13 +179,14 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
} }
} }
public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) { public void send(final MQDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) {
// 获取当前topic的分区数 // 获取当前topic的分区数
Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
destination.getDynamicTopicPartitionNum()); destination.getDynamicTopicPartitionNum());
// 获取topic的队列数为分区数 // 获取topic的队列数为分区数
if(partitionNum == null){ if (partitionNum == null) {
partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName); partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
} }
@@ -305,8 +304,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
} }
} }
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation") private void sendMessage(List<Message> messages, int partition) {
private void sendMessage(List<Message> messages, int partition) {
if (messages.isEmpty()) { if (messages.isEmpty()) {
return; return;
} }
@@ -348,8 +346,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
} }
} }
@Override @Override public void stop() {
public void stop() {
logger.info("## Stop RocketMQ producer##"); logger.info("## Stop RocketMQ producer##");
this.defaultMQProducer.shutdown(); this.defaultMQProducer.shutdown();
if (sendPartitionExecutor != null) { if (sendPartitionExecutor != null) {
@@ -359,14 +356,14 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
super.stop(); super.stop();
} }
private Integer getTopicDynamicQueuesSize(Boolean enable, String topicName){ private Integer getTopicDynamicQueuesSize(Boolean enable, String topicName) {
if(enable!=null && enable){ if (enable != null && enable) {
topicName = this.defaultMQProducer.withNamespace(topicName); topicName = this.defaultMQProducer.withNamespace(topicName);
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl(); DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topicName); TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topicName);
if(topicInfo == null){ if (topicInfo == null) {
return null; return null;
}else{ } else {
return topicInfo.getMessageQueueList().size(); return topicInfo.getMessageQueueList().size();
} }
} }

View File

@@ -223,7 +223,7 @@ public class BinlogDownloadQueue {
TarArchiveEntry tarArchiveEntry = null; TarArchiveEntry tarArchiveEntry = null;
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) { while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
String name = tarArchiveEntry.getName(); String name = tarArchiveEntry.getName();
File tarFile = new File(parentFile, name); File tarFile = new File(parentFile, name + ".tmp");
logger.info("start to download file " + tarFile.getName()); logger.info("start to download file " + tarFile.getName());
if (tarFile.exists()) { if (tarFile.exists()) {
tarFile.delete(); tarFile.delete();
@@ -244,7 +244,7 @@ public class BinlogDownloadQueue {
} }
tais.close(); tais.close();
} else { } else {
File file = new File(parentFile, fileName); File file = new File(parentFile, fileName + ".tmp");
if (file.exists()) { if (file.exists()) {
file.delete(); file.delete();
} }