fixed compatible and code format

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQConstants.java

@@ -1,7 +1,5 @@
 package com.alibaba.otter.canal.connector.pulsarmq.config;
 
-import org.joda.time.Seconds;
-
 /**
  * PulsarMQ配置
  *
@@ -10,6 +8,7 @@ import org.joda.time.Seconds;
  * @since 1 by chad at 2021/9/15 新增配置文件
  */
 public class PulsarMQConstants {
+
     public static final String ROOT                               = "pulsarmq";
     /**
      * pulsar服务连接地址
@@ -62,5 +61,4 @@ public class PulsarMQConstants {
      */
     public static final String PULSARMQ_ADMIN_SERVER_URL          = ROOT + "." + "adminServerUrl";
 
-
 }

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQProducerConfig.java

@@ -4,11 +4,13 @@ import com.alibaba.otter.canal.connector.core.config.MQProperties;
 
 /**
  * Pulsar生产者配置
+ * 
  * @author chad
  * @date 2021/9/15 11:23
  * @since 1 by chad at 2021/9/15 新增
  */
 public class PulsarMQProducerConfig extends MQProperties {
+
     /**
      * pulsar服务连接地址
      * <p>

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumer.java

@@ -1,20 +1,24 @@
 package com.alibaba.otter.canal.connector.pulsarmq.consumer;
 
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.client.api.*;
+
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.otter.canal.common.utils.MQUtil;
 import com.alibaba.otter.canal.connector.core.config.CanalConstants;
 import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
 import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
 import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
+import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.pulsar.client.api.*;
-
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
 
 /**
  * Pulsar consumer SPI 实现
@@ -24,6 +28,7 @@ import java.util.concurrent.TimeUnit;
  */
 @SPI("pulsarmq")
 public class CanalPulsarMQConsumer implements CanalMsgConsumer {
+
     /**
      * 连接pulsar客户端
      */
@@ -58,8 +63,7 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
      */
     private int                       batchSize              = 30;
     /**
-     * 与{@code batchSize}一起决定批量获取的数据大小
+     * 与{@code batchSize}一起决定批量获取的数据大小 当：
-     * 当：
      * <p>
      * 1. {@code batchSize} 条消息未消费时<br/>
      * 2. 距上一次批量消费时间达到{@code batchTimeoutSeconds}秒时
@@ -106,9 +110,12 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
         }
         this.serviceUrl = properties.getProperty(PulsarMQConstants.PULSARMQ_SERVER_URL);
         this.roleToken = properties.getProperty(PulsarMQConstants.PULSARMQ_ROLE_TOKEN);
-        //this.subscriptName = properties.getProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME);
+        this.subscriptName = properties.getProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME);
         // 采用groupId作为subscriptName，避免所有的都是同一个订阅者名称
+        if (StringUtils.isEmpty(this.subscriptName)) {
             this.subscriptName = groupId;
+        }
+
         if (StringUtils.isEmpty(this.subscriptName)) {
             throw new RuntimeException("Pulsar Consumer subscriptName required");
         }
@@ -153,9 +160,9 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
         }
         // 连接创建客户端
         try {
-            //AuthenticationDataProvider
+            // AuthenticationDataProvider
             ClientBuilder builder = PulsarClient.builder().serviceUrl(serviceUrl);
-            if(StringUtils.isNotEmpty(roleToken)) {
+            if (StringUtils.isNotEmpty(roleToken)) {
                 builder.authentication(AuthenticationFactory.token(roleToken));
             }
             pulsarClient = builder.build();
@@ -177,8 +184,7 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
             // 调用consumer.negativeAcknowledge(message) （即nack）来表示消费失败的消息
             // 在指定的时间进行重新消费，默认是1分钟。
             .negativeAckRedeliveryDelay(this.redeliveryDelaySeconds, TimeUnit.SECONDS)
-                .subscriptionName(this.subscriptName)
+            .subscriptionName(this.subscriptName);
-        ;
         if (this.isRetry) {
             DeadLetterPolicy.DeadLetterPolicyBuilder dlqBuilder = DeadLetterPolicy.builder()
                 // 最大重试次数
@@ -191,17 +197,15 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
                 dlqBuilder.deadLetterTopic(dlqTopic);
             }
 
-            //默认关闭，如果需要重试则开启
+            // 默认关闭，如果需要重试则开启
-            builder.enableRetry(true)
+            builder.enableRetry(true).deadLetterPolicy(dlqBuilder.build());
-                    .deadLetterPolicy(dlqBuilder.build());
         }
 
         // ack超时
         builder.ackTimeout(this.ackTimeoutSeconds, TimeUnit.SECONDS);
 
         // pulsar批量获取消息设置
-        builder.batchReceivePolicy(new BatchReceivePolicy.Builder()
+        builder.batchReceivePolicy(new BatchReceivePolicy.Builder().maxNumMessages(this.batchSize)
-                .maxNumMessages(this.batchSize)
             .timeout(this.getBatchTimeoutSeconds, TimeUnit.SECONDS)
             .build());
 
@@ -225,17 +229,14 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
             this.lastGetBatchMessage = messages;
             for (org.apache.pulsar.client.api.Message<byte[]> msg : messages) {
                 byte[] data = msg.getData();
-                /*if (!this.flatMessage) {
+                if (!this.flatMessage) {
                     Message message = CanalMessageSerializerUtil.deserializer(data);
                     List<CommonMessage> list = MessageUtil.convert(message);
                     messageList.addAll(list);
                 } else {
                     CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
                     messageList.add(commonMessage);
-                }*/
+                }
-                // CanalMessageSerializerUtil.deserializer(data) 会转换失败
-                CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
-                messageList.add(commonMessage);
             }
         } catch (PulsarClientException e) {
             throw new CanalClientException("Receive pulsar batch message error", e);

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/producer/CanalPulsarMQProducer.java

@@ -1,5 +1,17 @@
 package com.alibaba.otter.canal.connector.pulsarmq.producer;
 
+import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.fastjson2.JSONWriter.Feature;
 import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
@@ -16,19 +28,6 @@ import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
 import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQProducerConfig;
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.FlatMessage;
-import org.apache.commons.lang.StringUtils;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.*;
-import org.apache.pulsar.shade.com.google.gson.JsonParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * PulsarMQ Producer SPI 实现
@@ -39,15 +38,14 @@ import java.util.stream.Collectors;
 @SPI("pulsarmq")
 public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQProducer {
 
-    private static final Logger logger = LoggerFactory.getLogger(CanalPulsarMQProducer.class);
-
-    private static final Map<String, Producer<byte[]>> PRODUCERS = new HashMap<>();
-
-    protected ThreadPoolExecutor sendPartitionExecutor;
     /**
      * 消息体分区属性名称
      */
     public static final String                         MSG_PROPERTY_PARTITION_NAME = "partitionNum";
+    private static final Logger                        logger                      = LoggerFactory
+        .getLogger(CanalPulsarMQProducer.class);
+    private static final Map<String, Producer<byte[]>> PRODUCERS                   = new HashMap<>();
+    protected ThreadPoolExecutor                       sendPartitionExecutor;
     /**
      * pulsar客户端，管理连接
      */
@@ -70,7 +68,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
             ClientBuilder builder = PulsarClient.builder()
                 // 填写pulsar的连接地址
                 .serviceUrl(pulsarMQProducerConfig.getServerUrl());
-            if(StringUtils.isNotEmpty(pulsarMQProducerConfig.getRoleToken())) {
+            if (StringUtils.isNotEmpty(pulsarMQProducerConfig.getRoleToken())) {
                 // 角色权限认证的token
                 builder.authentication(AuthenticationFactory.token(pulsarMQProducerConfig.getRoleToken()));
             }
@@ -80,7 +78,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         }
 
         // 初始化Pulsar admin
-        if(StringUtils.isNotEmpty(pulsarMQProducerConfig.getAdminServerUrl())) {
+        if (StringUtils.isNotEmpty(pulsarMQProducerConfig.getAdminServerUrl())) {
             try {
                 pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsarMQProducerConfig.getAdminServerUrl()).build();
             } catch (PulsarClientException e) {
@@ -119,12 +117,13 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         if (!StringUtils.isEmpty(roleToken)) {
             tmpProperties.setRoleToken(roleToken);
         }
-        String topicTenantPrefix = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_TOPIC_TENANT_PREFIX);
+        String topicTenantPrefix = PropertiesUtils.getProperty(properties,
+            PulsarMQConstants.PULSARMQ_TOPIC_TENANT_PREFIX);
         if (!StringUtils.isEmpty(topicTenantPrefix)) {
             tmpProperties.setTopicTenantPrefix(topicTenantPrefix);
         }
         String adminServerUrl = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_ADMIN_SERVER_URL);
-        if(!StringUtils.isEmpty(adminServerUrl)) {
+        if (!StringUtils.isEmpty(adminServerUrl)) {
             tmpProperties.setAdminServerUrl(adminServerUrl);
         }
         if (logger.isDebugEnabled()) {
@@ -154,9 +153,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         try {
             if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
                 // 动态topic
-                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
+                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
-                        destination.getTopic(),
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
-                        destination.getDynamicTopic());
 
                 for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
                     String topicName = entry.getKey().replace('.', '_');
@@ -195,7 +193,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
      * @author chad
      * @since 1.0.0 by chad at 2021/9/2: 新增
      */
-    public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
+    public void send(final MQDestination destination, String topicName,
+                     com.alibaba.otter.canal.protocol.Message message) {
 
         // 获取当前topic的分区数
         Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
@@ -204,7 +203,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
             partitionNum = destination.getPartitionsNum();
         }
         // 创建多分区topic
-        if(pulsarAdmin!=null && partitionNum!=null && partitionNum>0 && PRODUCERS.get(topicName)==null) {
+        if (pulsarAdmin != null && partitionNum != null && partitionNum > 0 && PRODUCERS.get(topicName) == null) {
             createMultipleTopic(topicName, partitionNum);
         }
 
@@ -304,7 +303,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         try {
             MessageId msgResultId = producer.newMessage()
                 .property(MSG_PROPERTY_PARTITION_NAME, String.valueOf(partitionNum))
-                    .value(msgBytes).send();
+                .value(msgBytes)
+                .send();
             // todo 判断发送结果
             if (logger.isDebugEnabled()) {
                 logger.debug("Send Message to topic:{} Result: {}", topic, msgResultId);
@@ -328,8 +328,7 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         Producer<byte[]> producer = getProducer(topic);
         for (FlatMessage f : flatMessages) {
             try {
-                MessageId msgResultId = producer
+                MessageId msgResultId = producer.newMessage()
-                        .newMessage()
                     .property(MSG_PROPERTY_PARTITION_NAME, String.valueOf(partition))
                     .value(JSON.toJSONBytes(f, Feature.WriteNulls))
                     .send()
@@ -346,10 +345,11 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
 
     /**
      * 创建多分区topic
+     * 
      * @param topic
      * @param partitionNum
      */
-    private void createMultipleTopic(String topic,Integer partitionNum) {
+    private void createMultipleTopic(String topic, Integer partitionNum) {
         // 拼接topic前缀
         PulsarMQProducerConfig pulsarMQProperties = (PulsarMQProducerConfig) this.mqProperties;
         String prefix = pulsarMQProperties.getTopicTenantPrefix();
@@ -368,8 +368,10 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
             // TODO 无论是否报错，都继续后续的操作，此处不进行阻塞
         }
     }
+
     /**
      * 获取topic
+     * 
      * @param topic
      * @return
      */
@@ -412,6 +414,26 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         return producer;
     }
 
+    @Override
+    public void stop() {
+        logger.info("## Stop PulsarMQ producer##");
+
+        for (Producer p : PRODUCERS.values()) {
+            try {
+                if (null != p && p.isConnected()) {
+                    p.close();
+                }
+            } catch (PulsarClientException e) {
+                logger.warn("close producer name: {}, topic: {}, error: {}",
+                    p.getProducerName(),
+                    p.getTopic(),
+                    e.getMessage());
+            }
+        }
+
+        super.stop();
+    }
+
     /**
      * Pulsar自定义路由策略
      *
@@ -421,9 +443,10 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
      * @since 2 by chad at 2021/9/17 修改为msg自带目标分区
      */
     private static class MessageRouterImpl implements MessageRouter {
+
         private String topicLocal;
 
-        public MessageRouterImpl(String topicLocal) {
+        public MessageRouterImpl(String topicLocal){
             this.topicLocal = topicLocal;
         }
 
@@ -435,7 +458,8 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
                 try {
                     partition = Integer.parseInt(partitionStr);
                 } catch (NumberFormatException e) {
-                    logger.warn("Parse msg {} property failed for value: {}", MSG_PROPERTY_PARTITION_NAME, partitionStr);
+                    logger
+                        .warn("Parse msg {} property failed for value: {}", MSG_PROPERTY_PARTITION_NAME, partitionStr);
                 }
             }
             // topic创建时设置的分区数
@@ -447,21 +471,4 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
             return partition;
         }
     }
-
-    @Override
-    public void stop() {
-        logger.info("## Stop PulsarMQ producer##");
-
-        for (Producer p : PRODUCERS.values()) {
-            try {
-                if (null != p && p.isConnected()) {
-                    p.close();
-                }
-            } catch (PulsarClientException e) {
-                logger.warn("close producer name: {}, topic: {}, error: {}", p.getProducerName(), p.getTopic(), e.getMessage());
-            }
-        }
-
-        super.stop();
-    }
 }

connector/pulsarmq-connector/src/test/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumerTest.java

@@ -1,23 +1,19 @@
 package com.alibaba.otter.canal.connector.pulsarmq.consumer;
 
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.otter.canal.connector.core.config.CanalConstants;
 import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
-import org.apache.commons.lang.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.time.format.DateTimeFormatter;
-import java.util.List;
-import java.util.Properties;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.*;
 
 /**
  * @author chad