chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加canal-kafka-client的running状态监控

Browse Source
This commit is contained in:
rewerma
2018-06-21 11:30:11 +08:00
parent f867b183ab
commit f2e3ff2dae
9 changed files with 553 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
kafka-client/src/test/java/com/alibaba/otter/canal/kafka/client/running/AbstractKafkaTest.java
View File

@@ -12,8 +12,9 @@ public abstract class AbstractKafkaTest {
public static String topic = "example";
public static Integer partition = null;
public static String groupId = "g1";
public static String groupId = "g4";
public static String servers = "slave1:6667,slave2:6667,slave3:6667";
public static String zkServers = "slave1:2181,slave2:2181,slave3:2181";
public void sleep(long time) {
try {

85
kafka-client/src/test/java/com/alibaba/otter/canal/kafka/client/running/CanalKafkaClientExample.java
View File

@@ -19,31 +19,32 @@ import com.alibaba.otter.canal.protocol.Message;
*/
public class CanalKafkaClientExample {
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
private KafkaCanalConnector connector;
private KafkaCanalConnector connector;
private static volatile boolean running = false;
private static volatile boolean running = false;
private Thread thread = null;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientExample(String servers, String topic, Integer partition, String groupId){
connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId);
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId) {
connector = KafkaCanalConnectors.newKafkaConnector(zkServers, servers, topic, partition, groupId);
}
public static void main(String[] args) {
try {
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
@@ -86,6 +87,7 @@ public class CanalKafkaClientExample {
if (!running) {
return;
}
connector.stopRunning();
running = false;
if (thread != null) {
try {
@@ -99,34 +101,37 @@ public class CanalKafkaClientExample {
private void process() {
while (!running)
;
try {
connector.subscribe();
while (running) {
try {
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
if (message == null) {
continue;
}
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
} else {
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
}
while (running) {
try {
connector.connect();
connector.subscribe();
while (running) {
try {
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
if (message == null) {
continue;
}
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
} else {
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
}
connector.ack(); // 提交确认
} catch (Exception e) {
logger.error(e.getMessage(), e);
connector.ack(); // 提交确认
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
try {
@@ -134,6 +139,6 @@ public class CanalKafkaClientExample {
} catch (WakeupException e) {
// No-op. Continue process
}
connector.close();
connector.disconnnect();
}
}

3
kafka-client/src/test/java/com/alibaba/otter/canal/kafka/client/running/KafkaClientRunningTest.java
View File

@@ -35,6 +35,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
@Override
public void run() {
connector.connect();
connector.subscribe();
while (running) {
try {
@@ -48,7 +49,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
}
}
connector.unsubscribe();
connector.close();
connector.disconnnect();
}
});