chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed compatible and code format

Browse Source
This commit is contained in:
jianghang.loujh
2022-05-13 11:59:57 +08:00
parent 25be0259a4
commit 33733aef25
5 changed files with 167 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQConstants.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.connector.pulsarmq.config;
import org.joda.time.Seconds;
/**
* PulsarMQ配置
*
@@ -10,6 +8,7 @@ import org.joda.time.Seconds;
* @since 1 by chad at 2021/9/15 新增配置文件
*/
public class PulsarMQConstants {
public static final String ROOT = "pulsarmq";
/**
* pulsar服务连接地址
@@ -62,5 +61,4 @@ public class PulsarMQConstants {
*/
public static final String PULSARMQ_ADMIN_SERVER_URL = ROOT + "." + "adminServerUrl";
}

2
connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQProducerConfig.java
View File

@@ -4,11 +4,13 @@ import com.alibaba.otter.canal.connector.core.config.MQProperties;
/**
* Pulsar生产者配置
*
* @author chad
* @date 2021/9/15 11:23
* @since 1 by chad at 2021/9/15 新增
*/
public class PulsarMQProducerConfig extends MQProperties {
/**
* pulsar服务连接地址
* <p>

47
connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumer.java
View File

@@ -1,20 +1,24 @@
package com.alibaba.otter.canal.connector.pulsarmq.consumer;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.pulsar.client.api.*;
import com.alibaba.fastjson2.JSON;
import com.alibaba.otter.canal.common.utils.MQUtil;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.SPI;
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.pulsar.client.api.*;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
/**
* Pulsar consumer SPI 实现
@@ -24,6 +28,7 @@ import java.util.concurrent.TimeUnit;
*/
@SPI("pulsarmq")
public class CanalPulsarMQConsumer implements CanalMsgConsumer {
/**
* 连接pulsar客户端
*/
@@ -58,8 +63,7 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
*/
private int batchSize = 30;
/**
* 与{@code batchSize}一起决定批量获取的数据大小
* 当：
* 与{@code batchSize}一起决定批量获取的数据大小 当：
* <p>
* 1. {@code batchSize} 条消息未消费时<br/>
* 2. 距上一次批量消费时间达到{@code batchTimeoutSeconds}秒时
@@ -106,9 +110,12 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
}
this.serviceUrl = properties.getProperty(PulsarMQConstants.PULSARMQ_SERVER_URL);
this.roleToken = properties.getProperty(PulsarMQConstants.PULSARMQ_ROLE_TOKEN);
//this.subscriptName = properties.getProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME);
this.subscriptName = properties.getProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME);
// 采用groupId作为subscriptName，避免所有的都是同一个订阅者名称
if (StringUtils.isEmpty(this.subscriptName)) {
this.subscriptName = groupId;
}
if (StringUtils.isEmpty(this.subscriptName)) {
throw new RuntimeException("Pulsar Consumer subscriptName required");
}
@@ -153,9 +160,9 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
}
// 连接创建客户端
try {
//AuthenticationDataProvider
// AuthenticationDataProvider
ClientBuilder builder = PulsarClient.builder().serviceUrl(serviceUrl);
if(StringUtils.isNotEmpty(roleToken)) {
if (StringUtils.isNotEmpty(roleToken)) {
builder.authentication(AuthenticationFactory.token(roleToken));
}
pulsarClient = builder.build();
@@ -177,8 +184,7 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
// 调用consumer.negativeAcknowledge(message) （即nack）来表示消费失败的消息
// 在指定的时间进行重新消费，默认是1分钟。
.negativeAckRedeliveryDelay(this.redeliveryDelaySeconds, TimeUnit.SECONDS)
.subscriptionName(this.subscriptName)
;
.subscriptionName(this.subscriptName);
if (this.isRetry) {
DeadLetterPolicy.DeadLetterPolicyBuilder dlqBuilder = DeadLetterPolicy.builder()
// 最大重试次数
@@ -191,17 +197,15 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
dlqBuilder.deadLetterTopic(dlqTopic);
}
//默认关闭，如果需要重试则开启
builder.enableRetry(true)
.deadLetterPolicy(dlqBuilder.build());
// 默认关闭，如果需要重试则开启
builder.enableRetry(true).deadLetterPolicy(dlqBuilder.build());
}
// ack超时
builder.ackTimeout(this.ackTimeoutSeconds, TimeUnit.SECONDS);
// pulsar批量获取消息设置
builder.batchReceivePolicy(new BatchReceivePolicy.Builder()
.maxNumMessages(this.batchSize)
builder.batchReceivePolicy(new BatchReceivePolicy.Builder().maxNumMessages(this.batchSize)
.timeout(this.getBatchTimeoutSeconds, TimeUnit.SECONDS)
.build());
@@ -225,17 +229,14 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
this.lastGetBatchMessage = messages;
for (org.apache.pulsar.client.api.Message<byte[]> msg : messages) {
byte[] data = msg.getData();
/*if (!this.flatMessage) {
if (!this.flatMessage) {
Message message = CanalMessageSerializerUtil.deserializer(data);
List<CommonMessage> list = MessageUtil.convert(message);
messageList.addAll(list);
} else {
CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
messageList.add(commonMessage);
}*/
// CanalMessageSerializerUtil.deserializer(data) 会转换失败
CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
messageList.add(commonMessage);
}
}
} catch (PulsarClientException e) {
throw new CanalClientException("Receive pulsar batch message error", e);

107
connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/producer/CanalPulsarMQProducer.java
View File

@@ -1,5 +1,17 @@
package com.alibaba.otter.canal.connector.pulsarmq.producer;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONWriter.Feature;
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
@@ -16,19 +28,6 @@ import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQProducerConfig;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage;
import org.apache.commons.lang.StringUtils;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.*;
import org.apache.pulsar.shade.com.google.gson.JsonParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* PulsarMQ Producer SPI 实现
@@ -39,15 +38,14 @@ import java.util.stream.Collectors;
@SPI("pulsarmq")
public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalPulsarMQProducer.class);
private static final Map<String, Producer<byte[]>> PRODUCERS = new HashMap<>();
protected ThreadPoolExecutor sendPartitionExecutor;
/**
* 消息体分区属性名称
*/
public static final String MSG_PROPERTY_PARTITION_NAME = "partitionNum";
private static final Logger logger = LoggerFactory
.getLogger(CanalPulsarMQProducer.class);
private static final Map<String, Producer<byte[]>> PRODUCERS = new HashMap<>();
protected ThreadPoolExecutor sendPartitionExecutor;
/**
* pulsar客户端，管理连接
*/
@@ -70,7 +68,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
ClientBuilder builder = PulsarClient.builder()
// 填写pulsar的连接地址
.serviceUrl(pulsarMQProducerConfig.getServerUrl());
if(StringUtils.isNotEmpty(pulsarMQProducerConfig.getRoleToken())) {
if (StringUtils.isNotEmpty(pulsarMQProducerConfig.getRoleToken())) {
// 角色权限认证的token
builder.authentication(AuthenticationFactory.token(pulsarMQProducerConfig.getRoleToken()));
}
@@ -80,7 +78,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
}
// 初始化Pulsar admin
if(StringUtils.isNotEmpty(pulsarMQProducerConfig.getAdminServerUrl())) {
if (StringUtils.isNotEmpty(pulsarMQProducerConfig.getAdminServerUrl())) {
try {
pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsarMQProducerConfig.getAdminServerUrl()).build();
} catch (PulsarClientException e) {
@@ -119,12 +117,13 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
if (!StringUtils.isEmpty(roleToken)) {
tmpProperties.setRoleToken(roleToken);
}
String topicTenantPrefix = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_TOPIC_TENANT_PREFIX);
String topicTenantPrefix = PropertiesUtils.getProperty(properties,
PulsarMQConstants.PULSARMQ_TOPIC_TENANT_PREFIX);
if (!StringUtils.isEmpty(topicTenantPrefix)) {
tmpProperties.setTopicTenantPrefix(topicTenantPrefix);
}
String adminServerUrl = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_ADMIN_SERVER_URL);
if(!StringUtils.isEmpty(adminServerUrl)) {
if (!StringUtils.isEmpty(adminServerUrl)) {
tmpProperties.setAdminServerUrl(adminServerUrl);
}
if (logger.isDebugEnabled()) {
@@ -154,9 +153,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
try {
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
destination.getTopic(),
destination.getDynamicTopic());
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
@@ -195,7 +193,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
* @author chad
* @since 1.0.0 by chad at 2021/9/2: 新增
*/
public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
public void send(final MQDestination destination, String topicName,
com.alibaba.otter.canal.protocol.Message message) {
// 获取当前topic的分区数
Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
@@ -204,7 +203,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
partitionNum = destination.getPartitionsNum();
}
// 创建多分区topic
if(pulsarAdmin!=null && partitionNum!=null && partitionNum>0 && PRODUCERS.get(topicName)==null) {
if (pulsarAdmin != null && partitionNum != null && partitionNum > 0 && PRODUCERS.get(topicName) == null) {
createMultipleTopic(topicName, partitionNum);
}
@@ -304,7 +303,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
try {
MessageId msgResultId = producer.newMessage()
.property(MSG_PROPERTY_PARTITION_NAME, String.valueOf(partitionNum))
.value(msgBytes).send();
.value(msgBytes)
.send();
// todo 判断发送结果
if (logger.isDebugEnabled()) {
logger.debug("Send Message to topic:{} Result: {}", topic, msgResultId);
@@ -328,8 +328,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
Producer<byte[]> producer = getProducer(topic);
for (FlatMessage f : flatMessages) {
try {
MessageId msgResultId = producer
.newMessage()
MessageId msgResultId = producer.newMessage()
.property(MSG_PROPERTY_PARTITION_NAME, String.valueOf(partition))
.value(JSON.toJSONBytes(f, Feature.WriteNulls))
.send()
@@ -346,10 +345,11 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
/**
* 创建多分区topic
*
* @param topic
* @param partitionNum
*/
private void createMultipleTopic(String topic,Integer partitionNum) {
private void createMultipleTopic(String topic, Integer partitionNum) {
// 拼接topic前缀
PulsarMQProducerConfig pulsarMQProperties = (PulsarMQProducerConfig) this.mqProperties;
String prefix = pulsarMQProperties.getTopicTenantPrefix();
@@ -368,8 +368,10 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
// TODO 无论是否报错，都继续后续的操作，此处不进行阻塞
}
}
/**
* 获取topic
*
* @param topic
* @return
*/
@@ -412,6 +414,26 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
return producer;
}
@Override
public void stop() {
logger.info("## Stop PulsarMQ producer##");
for (Producer p : PRODUCERS.values()) {
try {
if (null != p && p.isConnected()) {
p.close();
}
} catch (PulsarClientException e) {
logger.warn("close producer name: {}, topic: {}, error: {}",
p.getProducerName(),
p.getTopic(),
e.getMessage());
}
}
super.stop();
}
/**
* Pulsar自定义路由策略
*
@@ -421,9 +443,10 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
* @since 2 by chad at 2021/9/17 修改为msg自带目标分区
*/
private static class MessageRouterImpl implements MessageRouter {
private String topicLocal;
public MessageRouterImpl(String topicLocal) {
public MessageRouterImpl(String topicLocal){
this.topicLocal = topicLocal;
}
@@ -435,7 +458,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
try {
partition = Integer.parseInt(partitionStr);
} catch (NumberFormatException e) {
logger.warn("Parse msg {} property failed for value: {}", MSG_PROPERTY_PARTITION_NAME, partitionStr);
logger
.warn("Parse msg {} property failed for value: {}", MSG_PROPERTY_PARTITION_NAME, partitionStr);
}
}
// topic创建时设置的分区数
@@ -447,21 +471,4 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
return partition;
}
}
@Override
public void stop() {
logger.info("## Stop PulsarMQ producer##");
for (Producer p : PRODUCERS.values()) {
try {
if (null != p && p.isConnected()) {
p.close();
}
} catch (PulsarClientException e) {
logger.warn("close producer name: {}, topic: {}, error: {}", p.getProducerName(), p.getTopic(), e.getMessage());
}
}
super.stop();
}
}

24
connector/pulsarmq-connector/src/test/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumerTest.java
View File

@@ -1,23 +1,19 @@
package com.alibaba.otter.canal.connector.pulsarmq.consumer;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.fastjson2.JSON;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
import org.apache.commons.lang.StringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.*;
/**
* @author chad