mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
Merge canal repo master
This commit is contained in:
duhengforever
374
client/pom.xml
374
client/pom.xml
@@ -1,115 +1,265 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.1.1-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal client module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- 客户端要使用请单独引入kafka依赖 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-client</artifactId>
|
||||
<version>4.3.0</version>
|
||||
</dependency>
|
||||
<!-- junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.1.1-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal client module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
</profile>
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</dependency>
|
||||
|
||||
<profile>
|
||||
<id>javadoc</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<aggregate>true</aggregate>
|
||||
<show>public</show>
|
||||
<nohelp>true</nohelp>
|
||||
<header>${project.artifactId}-${project.version}</header>
|
||||
<footer>${project.artifactId}-${project.version}</footer>
|
||||
<doctitle>${project.artifactId}-${project.version}</doctitle>
|
||||
<links>
|
||||
<link>https://github.com/alibaba/canal</link>
|
||||
</links>
|
||||
<outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}
|
||||
</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-scm-publish-plugin</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>publish-scm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
|
||||
<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}
|
||||
</checkinComment>
|
||||
<content>${project.build.directory}/apidocs</content>
|
||||
<skipDeletedFiles>true</skipDeletedFiles>
|
||||
<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
|
||||
<scmBranch>gh-pages</scmBranch>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
</dependency>
|
||||
<!-- zk -->
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.101tec</groupId>
|
||||
<artifactId>zkclient</artifactId>
|
||||
</dependency>
|
||||
<!-- external -->
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
<!-- log -->
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>jcl-over-slf4j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-aop</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-orm</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<version>${spring_version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-client</artifactId>
|
||||
<version>4.3.0</version>
|
||||
</dependency>
|
||||
<!-- 客户端要使用请单独引入kafka依赖 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
<artifactSet>
|
||||
<excludes>
|
||||
<exclude>org.apache.zookeeper:zookeeper</exclude>
|
||||
<exclude>com.101tec:zkclient</exclude>
|
||||
<exclude>commons-io:commons-io</exclude>
|
||||
<exclude>commons-lang:commons-lang</exclude>
|
||||
<exclude>commons-codec:commons-codec</exclude>
|
||||
<exclude>com.alibaba:fastjson</exclude>
|
||||
<exclude>ch.qos.logback:logback-core</exclude>
|
||||
<exclude>ch.qos.logback:logback-classic</exclude>
|
||||
<exclude>org.slf4j:jcl-over-slf4j</exclude>
|
||||
<exclude>org.slf4j:slf4j-api</exclude>
|
||||
<exclude>org.springframework:*</exclude>
|
||||
<exclude>org.jboss.netty:netty</exclude>
|
||||
<exclude>commons-logging:commons-logging</exclude>
|
||||
<exclude>aopalliance:aopalliance</exclude>
|
||||
<exclude>com.google.protobuf:protobuf-java</exclude>
|
||||
<exclude>io.netty:*</exclude>
|
||||
<exclude>junit:junit</exclude>
|
||||
</excludes>
|
||||
</artifactSet>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<pattern>com.google.guava</pattern>
|
||||
<shadedPattern>com.alibaba.google.guava</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.joda</pattern>
|
||||
<shadedPattern>com.alibaba.google.joda</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.common</pattern>
|
||||
<shadedPattern>com.alibaba.google.common</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.thirdparty</pattern>
|
||||
<shadedPattern>com.alibaba.google.thirdparty</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
<transformers>
|
||||
<transformer
|
||||
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>javadoc</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<aggregate>true</aggregate>
|
||||
<show>public</show>
|
||||
<nohelp>true</nohelp>
|
||||
<header>${project.artifactId}-${project.version}</header>
|
||||
<footer>${project.artifactId}-${project.version}</footer>
|
||||
<doctitle>${project.artifactId}-${project.version}</doctitle>
|
||||
<links>
|
||||
<link>https://github.com/alibaba/canal</link>
|
||||
</links>
|
||||
<outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-scm-publish-plugin</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>publish-scm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
|
||||
<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}</checkinComment>
|
||||
<content>${project.build.directory}/apidocs</content>
|
||||
<skipDeletedFiles>true</skipDeletedFiles>
|
||||
<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
|
||||
<scmBranch>gh-pages</scmBranch>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
@@ -8,8 +8,8 @@ import com.google.protobuf.ByteString;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class DeserializerUtil {
|
||||
private static Logger logger = LoggerFactory.getLogger(DeserializerUtil.class);
|
||||
public class CanalMessageDeserializer {
|
||||
private static Logger logger = LoggerFactory.getLogger(CanalMessageDeserializer.class);
|
||||
|
||||
public static Message deserializer(byte[] data) {
|
||||
try {
|
||||
@@ -19,9 +19,9 @@ public class DeserializerUtil {
|
||||
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
|
||||
switch (p.getType()) {
|
||||
case MESSAGES: {
|
||||
if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
|
||||
throw new CanalClientException("compression is not supported in this connector");
|
||||
}
|
||||
// if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
|
||||
// throw new CanalClientException("compression is not supported in this connector");
|
||||
// }
|
||||
|
||||
CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
|
||||
Message result = new Message(messages.getBatchId());
|
||||
@@ -159,7 +159,7 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
}
|
||||
//
|
||||
Handshake handshake = Handshake.parseFrom(p.getBody());
|
||||
supportedCompressions.addAll(handshake.getSupportedCompressionsList());
|
||||
supportedCompressions.add(handshake.getSupportedCompressions());
|
||||
//
|
||||
ClientAuth ca = ClientAuth.newBuilder()
|
||||
.setUsername(username != null ? username : "")
|
||||
@@ -180,8 +180,8 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
|
||||
Ack ackBody = Ack.parseFrom(ack.getBody());
|
||||
if (ackBody.getErrorCode() > 0) {
|
||||
throw new CanalClientException("something goes wrong when doing authentication: "
|
||||
+ ackBody.getErrorMessage());
|
||||
throw new CanalClientException(
|
||||
"something goes wrong when doing authentication: " + ackBody.getErrorMessage());
|
||||
}
|
||||
|
||||
connected = true;
|
||||
@@ -323,7 +323,8 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
Packet p = Packet.parseFrom(data);
|
||||
switch (p.getType()) {
|
||||
case MESSAGES: {
|
||||
if (!p.getCompression().equals(Compression.NONE)) {
|
||||
if (!p.getCompression().equals(Compression.NONE)
|
||||
&& !p.getCompression().equals(Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
|
||||
throw new CanalClientException("compression is not supported in this connector");
|
||||
}
|
||||
|
||||
@@ -360,11 +361,8 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
.setBatchId(batchId)
|
||||
.build();
|
||||
try {
|
||||
writeWithHeader(Packet.newBuilder()
|
||||
.setType(PacketType.CLIENTACK)
|
||||
.setBody(ca.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
writeWithHeader(
|
||||
Packet.newBuilder().setType(PacketType.CLIENTACK).setBody(ca.toByteString()).build().toByteArray());
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
package com.alibaba.otter.canal.client.kafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
||||
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.serialization.StringDeserializer;
|
||||
|
||||
import com.alibaba.otter.canal.client.kafka.running.ClientRunningData;
|
||||
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
|
||||
/**
|
||||
* canal kafka 数据操作客户端
|
||||
@@ -24,18 +25,19 @@ import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
public class KafkaCanalConnector {
|
||||
|
||||
private KafkaConsumer<String, Message> kafkaConsumer;
|
||||
private KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
private Properties properties;
|
||||
// private ClientRunningMonitor runningMonitor; // 运行控制
|
||||
// private BooleanMutex mutex = new BooleanMutex(false);
|
||||
private ZkClientx zkClientx;
|
||||
private volatile boolean connected = false;
|
||||
private volatile boolean running = false;
|
||||
private boolean flatMessage;
|
||||
|
||||
public KafkaCanalConnector(String zkServers, String servers, String topic, Integer partition, String groupId){
|
||||
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId,
|
||||
boolean flatMessage){
|
||||
this.topic = topic;
|
||||
this.partition = partition;
|
||||
this.flatMessage = flatMessage;
|
||||
|
||||
properties = new Properties();
|
||||
properties.put("bootstrap.servers", servers);
|
||||
@@ -45,32 +47,13 @@ public class KafkaCanalConnector {
|
||||
properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
|
||||
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
|
||||
properties.put("session.timeout.ms", "30000"); // 默认为30秒
|
||||
properties.put("max.poll.records", "1"); // 所以一次只取一条数据
|
||||
properties.put("max.poll.records", "100");
|
||||
properties.put("key.deserializer", StringDeserializer.class.getName());
|
||||
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
||||
|
||||
if (zkServers != null) {
|
||||
zkClientx = new ZkClientx(zkServers);
|
||||
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setGroupId(groupId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
|
||||
// runningMonitor = new ClientRunningMonitor();
|
||||
// runningMonitor.setTopic(topic);
|
||||
// runningMonitor.setZkClient(zkClientx);
|
||||
// runningMonitor.setClientData(clientData);
|
||||
// runningMonitor.setListener(new ClientRunningListener() {
|
||||
// public void processActiveEnter() {
|
||||
// mutex.set(true);
|
||||
// }
|
||||
//
|
||||
// public void processActiveExit() {
|
||||
// mutex.set(false);
|
||||
// }
|
||||
// });
|
||||
if (!flatMessage) {
|
||||
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
||||
} else {
|
||||
properties.put("value.deserializer", StringDeserializer.class.getName());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -93,57 +76,36 @@ public class KafkaCanalConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
// if (runningMonitor != null) {
|
||||
// if (!runningMonitor.isStart()) {
|
||||
// runningMonitor.start();
|
||||
// }
|
||||
// }
|
||||
|
||||
connected = true;
|
||||
|
||||
if (kafkaConsumer == null) {
|
||||
if (kafkaConsumer == null && !flatMessage) {
|
||||
kafkaConsumer = new KafkaConsumer<String, Message>(properties);
|
||||
}
|
||||
if (kafkaConsumer2 == null && flatMessage) {
|
||||
kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭链接
|
||||
*/
|
||||
public void disconnnect() {
|
||||
kafkaConsumer.close();
|
||||
public void disconnect() {
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.close();
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.close();
|
||||
}
|
||||
|
||||
connected = false;
|
||||
// if (runningMonitor.isStart()) {
|
||||
// runningMonitor.stop();
|
||||
// }
|
||||
}
|
||||
|
||||
private void waitClientRunning() {
|
||||
try {
|
||||
if (zkClientx != null) {
|
||||
if (!connected) {// 未调用connect
|
||||
throw new CanalClientException("should connect first");
|
||||
}
|
||||
|
||||
running = true;
|
||||
// mutex.get();// 阻塞等待
|
||||
} else {
|
||||
// 单机模式直接设置为running
|
||||
running = true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
running = true;
|
||||
}
|
||||
|
||||
public boolean checkValid() {
|
||||
if (zkClientx != null) {
|
||||
// return mutex.state();
|
||||
return true;
|
||||
} else {
|
||||
return true;// 默认都放过
|
||||
}
|
||||
return true;// 默认都放过
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -156,10 +118,20 @@ public class KafkaCanalConnector {
|
||||
}
|
||||
|
||||
if (partition == null) {
|
||||
kafkaConsumer.subscribe(Collections.singletonList(topic));
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.subscribe(Collections.singletonList(topic));
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.subscribe(Collections.singletonList(topic));
|
||||
}
|
||||
} else {
|
||||
TopicPartition topicPartition = new TopicPartition(topic, partition);
|
||||
kafkaConsumer.assign(Collections.singletonList(topicPartition));
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.assign(Collections.singletonList(topicPartition));
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.assign(Collections.singletonList(topicPartition));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +144,12 @@ public class KafkaCanalConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
kafkaConsumer.unsubscribe();
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.unsubscribe();
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.unsubscribe();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -180,22 +157,22 @@ public class KafkaCanalConnector {
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Message get() {
|
||||
public List<Message> get() {
|
||||
return get(100L, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public Message get(Long timeout, TimeUnit unit) {
|
||||
public List<Message> get(Long timeout, TimeUnit unit) {
|
||||
waitClientRunning();
|
||||
if (!running) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Message message = getWithoutAck(timeout, unit);
|
||||
List<Message> messages = getWithoutAck(timeout, unit);
|
||||
this.ack();
|
||||
return message;
|
||||
return messages;
|
||||
}
|
||||
|
||||
public Message getWithoutAck() {
|
||||
public List<Message> getWithoutAck() {
|
||||
return getWithoutAck(100L, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@@ -204,7 +181,7 @@ public class KafkaCanalConnector {
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Message getWithoutAck(Long timeout, TimeUnit unit) {
|
||||
public List<Message> getWithoutAck(Long timeout, TimeUnit unit) {
|
||||
waitClientRunning();
|
||||
if (!running) {
|
||||
return null;
|
||||
@@ -213,7 +190,33 @@ public class KafkaCanalConnector {
|
||||
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据
|
||||
|
||||
if (!records.isEmpty()) {
|
||||
return records.iterator().next().value();
|
||||
// return records.iterator().next().value();
|
||||
List<Message> messages = new ArrayList<>();
|
||||
for (ConsumerRecord<String, Message> record : records) {
|
||||
messages.add(record.value());
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<FlatMessage> getFlatMessageWithoutAck(Long timeout, TimeUnit unit) {
|
||||
waitClientRunning();
|
||||
if (!running) {
|
||||
return null;
|
||||
}
|
||||
|
||||
ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
|
||||
|
||||
if (!records.isEmpty()) {
|
||||
List<FlatMessage> flatMessages = new ArrayList<>();
|
||||
for (ConsumerRecord<String, String> record : records) {
|
||||
String flatMessageJson = record.value();
|
||||
FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
|
||||
flatMessages.add(flatMessage);
|
||||
}
|
||||
|
||||
return flatMessages;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -227,7 +230,12 @@ public class KafkaCanalConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
kafkaConsumer.commitSync();
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.commitSync();
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.commitAsync();
|
||||
}
|
||||
}
|
||||
|
||||
public void stopRunning() {
|
||||
|
||||
@@ -17,8 +17,9 @@ public class KafkaCanalConnectors {
|
||||
* @param groupId
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) {
|
||||
return new KafkaCanalConnector(null, servers, topic, partition, groupId);
|
||||
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition,
|
||||
String groupId) {
|
||||
return new KafkaCanalConnector(servers, topic, partition, groupId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -30,33 +31,21 @@ public class KafkaCanalConnectors {
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
|
||||
return new KafkaCanalConnector(null, servers, topic, null, groupId);
|
||||
return new KafkaCanalConnector(servers, topic, null, groupId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建kafka客户端链接
|
||||
*
|
||||
* @param zkServers
|
||||
* @param servers
|
||||
* @param topic
|
||||
* @param partition
|
||||
* @param groupId
|
||||
* @param flatMessage
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, Integer partition, String groupId) {
|
||||
return new KafkaCanalConnector(zkServers, servers, topic, partition, groupId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建kafka客户端链接
|
||||
*
|
||||
* @param zkServers
|
||||
* @param servers
|
||||
* @param topic
|
||||
* @param groupId
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, String groupId) {
|
||||
return new KafkaCanalConnector(zkServers, servers, topic, null, groupId);
|
||||
public static KafkaCanalConnector newKafkaConnector( String servers, String topic,
|
||||
Integer partition, String groupId,boolean flatMessage) {
|
||||
return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.kafka;
|
||||
|
||||
import com.alibaba.otter.canal.client.DeserializerUtil;
|
||||
import com.alibaba.otter.canal.client.CanalMessageDeserializer;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import java.util.Map;
|
||||
import org.apache.kafka.common.serialization.Deserializer;
|
||||
@@ -23,7 +23,7 @@ public class MessageDeserializer implements Deserializer<Message> {
|
||||
|
||||
@Override
|
||||
public Message deserialize(String topic1, byte[] data) {
|
||||
return DeserializerUtil.deserializer(data);
|
||||
return CanalMessageDeserializer.deserializer(data);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.alibaba.otter.canal.client.rocketmq;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.DeserializerUtil;
|
||||
import com.alibaba.otter.canal.client.CanalMessageDeserializer;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
import java.util.List;
|
||||
@@ -94,7 +94,7 @@ public class RocketMQCanalConnector implements CanalConnector {
|
||||
BlockingQueue<Message> messageList = new LinkedBlockingQueue<>();
|
||||
for (MessageExt messageExt : messageExts) {
|
||||
byte[] data = messageExt.getBody();
|
||||
Message message = DeserializerUtil.deserializer(data);
|
||||
Message message = CanalMessageDeserializer.deserializer(data);
|
||||
try {
|
||||
messageList.put(message);
|
||||
} catch (InterruptedException ex) {
|
||||
|
||||
@@ -20,117 +20,117 @@ import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||
|
||||
public class ClientRunningTest extends AbstractZkTest {
|
||||
|
||||
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||
private short clientId = 1001;
|
||||
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||
private short clientId = 1001;
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
@Before
|
||||
public void setUp() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
|
||||
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
|
||||
}
|
||||
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
}
|
||||
@After
|
||||
public void tearDown() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOneServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
|
||||
runningMonitor.start();
|
||||
sleep(2000L);
|
||||
runningMonitor.stop();
|
||||
sleep(2000L);
|
||||
@Test
|
||||
public void testOneServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
|
||||
runningMonitor.start();
|
||||
sleep(2000L);
|
||||
runningMonitor.stop();
|
||||
sleep(2000L);
|
||||
|
||||
if (countLatch.getCount() != 0) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
if (countLatch.getCount() != 0) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
|
||||
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
|
||||
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
|
||||
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||
executor.submit(new Runnable() {
|
||||
@Test
|
||||
public void testMultiServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
|
||||
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
|
||||
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
|
||||
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor1.isStart()) {
|
||||
runningMonitor1.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor1.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor1.isStart()) {
|
||||
runningMonitor1.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor1.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
executor.submit(new Runnable() {
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor2.isStart()) {
|
||||
runningMonitor2.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor2.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor2.isStart()) {
|
||||
runningMonitor2.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor2.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
executor.submit(new Runnable() {
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor3.isStart()) {
|
||||
runningMonitor3.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor3.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor3.isStart()) {
|
||||
runningMonitor3.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor3.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
sleep(30000L);
|
||||
}
|
||||
sleep(30000L);
|
||||
}
|
||||
|
||||
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
|
||||
final int port) {
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setClientId(clientId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
|
||||
final int port) {
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setClientId(clientId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
|
||||
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
|
||||
runningMonitor.setDestination(destination);
|
||||
runningMonitor.setZkClient(zkclientx);
|
||||
runningMonitor.setClientData(clientData);
|
||||
runningMonitor.setListener(new ClientRunningListener() {
|
||||
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
|
||||
runningMonitor.setDestination(destination);
|
||||
runningMonitor.setZkClient(zkclientx);
|
||||
runningMonitor.setClientData(clientData);
|
||||
runningMonitor.setListener(new ClientRunningListener() {
|
||||
|
||||
public InetSocketAddress processActiveEnter() {
|
||||
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
|
||||
countLatch.countDown();
|
||||
return new InetSocketAddress(AddressUtils.getHostIp(), port);
|
||||
}
|
||||
public InetSocketAddress processActiveEnter() {
|
||||
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
|
||||
countLatch.countDown();
|
||||
return new InetSocketAddress(AddressUtils.getHostIp(), port);
|
||||
}
|
||||
|
||||
public void processActiveExit() {
|
||||
countLatch.countDown();
|
||||
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
|
||||
}
|
||||
public void processActiveExit() {
|
||||
countLatch.countDown();
|
||||
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
|
||||
}
|
||||
|
||||
});
|
||||
runningMonitor.setDelayTime(1);
|
||||
return runningMonitor;
|
||||
}
|
||||
}
|
||||
});
|
||||
runningMonitor.setDelayTime(1);
|
||||
return runningMonitor;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.running.kafka;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.kafka.common.errors.WakeupException;
|
||||
@@ -19,32 +20,33 @@ import com.alibaba.otter.canal.protocol.Message;
|
||||
*/
|
||||
public class CanalKafkaClientExample {
|
||||
|
||||
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
|
||||
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
|
||||
|
||||
private KafkaCanalConnector connector;
|
||||
private KafkaCanalConnector connector;
|
||||
|
||||
private static volatile boolean running = false;
|
||||
private static volatile boolean running = false;
|
||||
|
||||
private Thread thread = null;
|
||||
private Thread thread = null;
|
||||
|
||||
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
|
||||
|
||||
public void uncaughtException(Thread t, Throwable e) {
|
||||
logger.error("parse events has an error", e);
|
||||
}
|
||||
};
|
||||
public void uncaughtException(Thread t, Throwable e) {
|
||||
logger.error("parse events has an error", e);
|
||||
}
|
||||
};
|
||||
|
||||
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId) {
|
||||
connector = KafkaCanalConnectors.newKafkaConnector(zkServers, servers, topic, partition, groupId);
|
||||
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
|
||||
connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
|
||||
AbstractKafkaTest.servers,
|
||||
AbstractKafkaTest.topic,
|
||||
AbstractKafkaTest.partition,
|
||||
AbstractKafkaTest.groupId);
|
||||
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
|
||||
AbstractKafkaTest.zkServers,
|
||||
AbstractKafkaTest.servers,
|
||||
AbstractKafkaTest.topic,
|
||||
AbstractKafkaTest.partition,
|
||||
AbstractKafkaTest.groupId);
|
||||
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
|
||||
kafkaCanalClientExample.start();
|
||||
logger.info("## the canal kafka consumer is running now ......");
|
||||
@@ -107,21 +109,24 @@ public class CanalKafkaClientExample {
|
||||
connector.subscribe();
|
||||
while (running) {
|
||||
try {
|
||||
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
|
||||
if (message == null) {
|
||||
List<Message> messages = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
|
||||
|
||||
if (messages == null) {
|
||||
continue;
|
||||
}
|
||||
long batchId = message.getId();
|
||||
int size = message.getEntries().size();
|
||||
if (batchId == -1 || size == 0) {
|
||||
// try {
|
||||
// Thread.sleep(1000);
|
||||
// } catch (InterruptedException e) {
|
||||
// }
|
||||
} else {
|
||||
// printSummary(message, batchId, size);
|
||||
// printEntry(message.getEntries());
|
||||
logger.info(message.toString());
|
||||
for (Message message : messages) {
|
||||
long batchId = message.getId();
|
||||
int size = message.getEntries().size();
|
||||
if (batchId == -1 || size == 0) {
|
||||
// try {
|
||||
// Thread.sleep(1000);
|
||||
// } catch (InterruptedException e) {
|
||||
// }
|
||||
} else {
|
||||
// printSummary(message, batchId, size);
|
||||
// printEntry(message.getEntries());
|
||||
logger.info(message.toString());
|
||||
}
|
||||
}
|
||||
|
||||
connector.ack(); // 提交确认
|
||||
@@ -139,6 +144,6 @@ public class CanalKafkaClientExample {
|
||||
} catch (WakeupException e) {
|
||||
// No-op. Continue process
|
||||
}
|
||||
connector.disconnnect();
|
||||
connector.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.running.kafka;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -39,9 +40,9 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
|
||||
connector.subscribe();
|
||||
while (running) {
|
||||
try {
|
||||
Message message = connector.getWithoutAck(3L, TimeUnit.SECONDS);
|
||||
if (message != null) {
|
||||
System.out.println(message);
|
||||
List<Message> messages = connector.getWithoutAck(3L, TimeUnit.SECONDS);
|
||||
if (messages != null) {
|
||||
System.out.println(messages);
|
||||
}
|
||||
connector.ack();
|
||||
} catch (WakeupException e) {
|
||||
@@ -49,7 +50,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
|
||||
}
|
||||
}
|
||||
connector.unsubscribe();
|
||||
connector.disconnnect();
|
||||
connector.disconnect();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user