chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge canal repo master

Browse Source
This commit is contained in:
duhengforever
2018-09-27 17:09:00 +08:00
parent a37a7ff142 ff592758cf
commit 5ae16b1c7c
73 changed files with 12936 additions and 10729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java
View File

@@ -20,117 +20,117 @@ import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
public class ClientRunningTest extends AbstractZkTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
private short clientId = 1001;
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
private short clientId = 1001;
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
}
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testOneServer() {
final CountDownLatch countLatch = new CountDownLatch(2);
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
runningMonitor.start();
sleep(2000L);
runningMonitor.stop();
sleep(2000L);
@Test
public void testOneServer() {
final CountDownLatch countLatch = new CountDownLatch(2);
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
runningMonitor.start();
sleep(2000L);
runningMonitor.stop();
sleep(2000L);
if (countLatch.getCount() != 0) {
Assert.fail();
}
}
if (countLatch.getCount() != 0) {
Assert.fail();
}
}
@Test
public void testMultiServer() {
final CountDownLatch countLatch = new CountDownLatch(30);
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
final ExecutorService executor = Executors.newFixedThreadPool(3);
executor.submit(new Runnable() {
@Test
public void testMultiServer() {
final CountDownLatch countLatch = new CountDownLatch(30);
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
final ExecutorService executor = Executors.newFixedThreadPool(3);
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor1.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
}
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor1.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
}
});
});
executor.submit(new Runnable() {
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor2.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
}
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor2.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
}
});
});
executor.submit(new Runnable() {
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor3.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
}
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor3.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
}
});
});
sleep(30000L);
}
sleep(30000L);
}
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
final int port) {
ClientRunningData clientData = new ClientRunningData();
clientData.setClientId(clientId);
clientData.setAddress(AddressUtils.getHostIp());
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
final int port) {
ClientRunningData clientData = new ClientRunningData();
clientData.setClientId(clientId);
clientData.setAddress(AddressUtils.getHostIp());
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
runningMonitor.setDestination(destination);
runningMonitor.setZkClient(zkclientx);
runningMonitor.setClientData(clientData);
runningMonitor.setListener(new ClientRunningListener() {
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
runningMonitor.setDestination(destination);
runningMonitor.setZkClient(zkclientx);
runningMonitor.setClientData(clientData);
runningMonitor.setListener(new ClientRunningListener() {
public InetSocketAddress processActiveEnter() {
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
countLatch.countDown();
return new InetSocketAddress(AddressUtils.getHostIp(), port);
}
public InetSocketAddress processActiveEnter() {
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
countLatch.countDown();
return new InetSocketAddress(AddressUtils.getHostIp(), port);
}
public void processActiveExit() {
countLatch.countDown();
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
}
public void processActiveExit() {
countLatch.countDown();
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
}
});
runningMonitor.setDelayTime(1);
return runningMonitor;
}
}
});
runningMonitor.setDelayTime(1);
return runningMonitor;
}
}

63
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.client.running.kafka;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.common.errors.WakeupException;
@@ -19,32 +20,33 @@ import com.alibaba.otter.canal.protocol.Message;
*/
public class CanalKafkaClientExample {
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
private KafkaCanalConnector connector;
private KafkaCanalConnector connector;
private static volatile boolean running = false;
private static volatile boolean running = false;
private Thread thread = null;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId) {
connector = KafkaCanalConnectors.newKafkaConnector(zkServers, servers, topic, partition, groupId);
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false);
}
public static void main(String[] args) {
try {
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
AbstractKafkaTest.zkServers,
AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,
AbstractKafkaTest.groupId);
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
@@ -107,21 +109,24 @@ public class CanalKafkaClientExample {
connector.subscribe();
while (running) {
try {
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
if (message == null) {
List<Message> messages = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
if (messages == null) {
continue;
}
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
} else {
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
for (Message message : messages) {
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// }
} else {
// printSummary(message, batchId, size);
// printEntry(message.getEntries());
logger.info(message.toString());
}
}
connector.ack(); // 提交确认
@@ -139,6 +144,6 @@ public class CanalKafkaClientExample {
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnnect();
connector.disconnect();
}
}

9
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.client.running.kafka;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -39,9 +40,9 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
connector.subscribe();
while (running) {
try {
Message message = connector.getWithoutAck(3L, TimeUnit.SECONDS);
if (message != null) {
System.out.println(message);
List<Message> messages = connector.getWithoutAck(3L, TimeUnit.SECONDS);
if (messages != null) {
System.out.println(messages);
}
connector.ack();
} catch (WakeupException e) {
@@ -49,7 +50,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
}
}
connector.unsubscribe();
connector.disconnnect();
connector.disconnect();
}
});