mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
init
This commit is contained in:
agapple
101
client/pom.xml
Normal file
101
client/pom.xml
Normal file
@@ -0,0 +1,101 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.0.19-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal client module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>javadoc</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<aggregate>true</aggregate>
|
||||
<show>public</show>
|
||||
<nohelp>true</nohelp>
|
||||
<header>${project.artifactId}-${project.version}</header>
|
||||
<footer>${project.artifactId}-${project.version}</footer>
|
||||
<doctitle>${project.artifactId}-${project.version}</doctitle>
|
||||
<links>
|
||||
<link>https://github.com/alibaba/canal</link>
|
||||
</links>
|
||||
<outputDirectory>${project.build.directory}/apidocs/apidocs/${pom.version}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-scm-publish-plugin</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>publish-scm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
|
||||
<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}</checkinComment>
|
||||
<content>${project.build.directory}/apidocs</content>
|
||||
<skipDeletedFiles>true</skipDeletedFiles>
|
||||
<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
|
||||
<scmBranch>gh-pages</scmBranch>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@@ -0,0 +1,156 @@
|
||||
package com.alibaba.otter.canal.client;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
|
||||
/**
|
||||
* canal数据操作客户端
|
||||
*
|
||||
* @author zebin.xuzb @ 2012-6-19
|
||||
* @author jianghang
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface CanalConnector {
|
||||
|
||||
/**
|
||||
* 链接对应的canal server
|
||||
*
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
public void connect() throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 释放链接
|
||||
*
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
public void disconnect() throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 检查下链接是否合法
|
||||
*
|
||||
* <pre>
|
||||
* 几种case下链接不合法:
|
||||
* 1. 链接canal server失败，一直没有一个可用的链接，返回false
|
||||
* 2. 当前客户端在进行running抢占的时候，做为备份节点存在，非处于工作节点，返回false
|
||||
*
|
||||
* 说明：
|
||||
* a. 当前客户端一旦做为备份节点存在，当前所有的对{@linkplain CanalConnector}的操作都会处于阻塞状态，直到转为工作节点
|
||||
* b. 所以业务方最好定时调用checkValid()方法用，比如调用CanalConnector所在线程的interrupt，直接退出CanalConnector，并根据自己的需要退出自己的资源
|
||||
* </pre>
|
||||
*
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
public boolean checkValid() throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 客户端订阅，重复订阅时会更新对应的filter信息
|
||||
*
|
||||
* <pre>
|
||||
* 说明：
|
||||
* a. 如果本次订阅中filter信息为空，则直接使用canal server服务端配置的filter信息
|
||||
* b. 如果本次订阅中filter信息不为空，目前会直接替换canal server服务端配置的filter信息，以本次提交的为准
|
||||
*
|
||||
* TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
|
||||
* </pre>
|
||||
*
|
||||
* @param clientIdentity
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
void subscribe(String filter) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 客户端订阅，不提交客户端filter，以服务端的filter为准
|
||||
*
|
||||
* @param clientIdentity
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
void subscribe() throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 取消订阅
|
||||
*
|
||||
* @param clientIdentity
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
void unsubscribe() throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 获取数据，自动进行确认，该方法返回的条件：尝试拿batchSize条记录，有多少取多少，不会阻塞等待
|
||||
*
|
||||
* @param batchSize
|
||||
* @return
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
Message get(int batchSize) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 获取数据，自动进行确认
|
||||
*
|
||||
* <pre>
|
||||
* 该方法返回的条件：
|
||||
* a. 拿够batchSize条记录或者超过timeout时间
|
||||
* b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
|
||||
* </pre>
|
||||
*
|
||||
* @param batchSize
|
||||
* @return
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 不指定 position 获取事件，该方法返回的条件: 尝试拿batchSize条记录，有多少取多少，不会阻塞等待<br/>
|
||||
* canal 会记住此 client 最新的position。 <br/>
|
||||
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
|
||||
*
|
||||
* @param batchSize
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
Message getWithoutAck(int batchSize) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 不指定 position 获取事件.
|
||||
*
|
||||
* <pre>
|
||||
* 该方法返回的条件：
|
||||
* a. 拿够batchSize条记录或者超过timeout时间
|
||||
* b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
|
||||
* </pre>
|
||||
*
|
||||
* canal 会记住此 client 最新的position。 <br/>
|
||||
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
|
||||
*
|
||||
* @param batchSize
|
||||
* @param timeout
|
||||
* @param unit
|
||||
* @return
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
|
||||
*
|
||||
* @param batchId
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
void ack(long batchId) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId
|
||||
*
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
void rollback(long batchId) throws CanalClientException;
|
||||
|
||||
/**
|
||||
* 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
|
||||
*
|
||||
* @throws CanalClientException
|
||||
*/
|
||||
void rollback() throws CanalClientException;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.alibaba.otter.canal.client;
|
||||
|
||||
import java.net.SocketAddress;
|
||||
import java.util.List;
|
||||
|
||||
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
|
||||
import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
|
||||
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
|
||||
import com.alibaba.otter.canal.client.impl.SimpleNodeAccessStrategy;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
|
||||
/**
|
||||
* canal connectors创建工具类
|
||||
*
|
||||
* @author jianghang 2012-10-29 下午11:18:50
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalConnectors {
|
||||
|
||||
/**
|
||||
* 创建单链接的客户端链接
|
||||
*
|
||||
* @param address
|
||||
* @param username
|
||||
* @param password
|
||||
* @return
|
||||
*/
|
||||
public static CanalConnector newSingleConnector(SocketAddress address, String destination, String username,
|
||||
String password) {
|
||||
SimpleCanalConnector canalConnector = new SimpleCanalConnector(address, username, password, destination);
|
||||
canalConnector.setSoTimeout(30 * 1000);
|
||||
return canalConnector;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建带cluster模式的客户端链接，自动完成failover切换
|
||||
*
|
||||
* @param addresses
|
||||
* @param username
|
||||
* @param password
|
||||
* @return
|
||||
*/
|
||||
public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
|
||||
String username, String password) {
|
||||
ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
|
||||
new SimpleNodeAccessStrategy(addresses));
|
||||
canalConnector.setSoTimeout(30 * 1000);
|
||||
return canalConnector;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描
|
||||
*
|
||||
* @param username
|
||||
* @param password
|
||||
* @return
|
||||
*/
|
||||
public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
|
||||
String password) {
|
||||
ClusterCanalConnector canalConnector = new ClusterCanalConnector(
|
||||
username,
|
||||
password,
|
||||
destination,
|
||||
new ClusterNodeAccessStrategy(
|
||||
destination,
|
||||
ZkClientx.getZkClient(zkServers)));
|
||||
canalConnector.setSoTimeout(30 * 1000);
|
||||
return canalConnector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.alibaba.otter.canal.client;
|
||||
|
||||
import java.net.SocketAddress;
|
||||
|
||||
/**
|
||||
* 集群节点访问控制接口
|
||||
*
|
||||
* @author jianghang 2012-10-29 下午07:55:41
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface CanalNodeAccessStrategy {
|
||||
|
||||
SocketAddress nextNode();
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package com.alibaba.otter.canal.client.impl;
|
||||
|
||||
import java.net.SocketAddress;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.exception.ExceptionUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
|
||||
/**
|
||||
* 集群版本connector实现，自带了failover功能<br/>
|
||||
*
|
||||
* @author jianghang 2012-10-29 下午08:04:06
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ClusterCanalConnector implements CanalConnector {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
private String username;
|
||||
private String password;
|
||||
private int soTimeout = 10000;
|
||||
private int retryTimes = 3;
|
||||
private int retryInterval = 5000; // 重试的时间间隔，默认5秒
|
||||
private CanalNodeAccessStrategy accessStrategy;
|
||||
private SimpleCanalConnector currentConnector;
|
||||
private String destination;
|
||||
private String filter; // 记录上一次的filter提交值,便于自动重试时提交
|
||||
|
||||
public ClusterCanalConnector(String username, String password, String destination,
|
||||
CanalNodeAccessStrategy accessStrategy){
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
this.destination = destination;
|
||||
this.accessStrategy = accessStrategy;
|
||||
}
|
||||
|
||||
public void connect() throws CanalClientException {
|
||||
while (currentConnector == null) {
|
||||
SocketAddress nextAddress = this.accessStrategy.nextNode();
|
||||
int times = 0;
|
||||
while (true) {
|
||||
try {
|
||||
currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination);
|
||||
currentConnector.setSoTimeout(soTimeout);
|
||||
if (filter != null) {
|
||||
currentConnector.setFilter(filter);
|
||||
}
|
||||
if (accessStrategy instanceof ClusterNodeAccessStrategy) {
|
||||
currentConnector.setZkClientx(((ClusterNodeAccessStrategy) accessStrategy).getZkClient());
|
||||
}
|
||||
|
||||
currentConnector.connect();
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
logger.warn("failed to connect to:{} after retry {} times", nextAddress, times);
|
||||
currentConnector.disconnect();
|
||||
currentConnector = null;
|
||||
// retry for #retryTimes for each node when trying to
|
||||
// connect to it.
|
||||
times = times + 1;
|
||||
if (times >= retryTimes) {
|
||||
throw new CanalClientException(e);
|
||||
} else {
|
||||
// fixed issue #55，增加sleep控制，避免重试connect时cpu使用过高
|
||||
try {
|
||||
Thread.sleep(retryInterval);
|
||||
} catch (InterruptedException e1) {
|
||||
throw new CanalClientException(e1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public boolean checkValid() {
|
||||
return currentConnector != null && currentConnector.checkValid();
|
||||
}
|
||||
|
||||
public void disconnect() throws CanalClientException {
|
||||
if (currentConnector != null) {
|
||||
currentConnector.disconnect();
|
||||
currentConnector = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void subscribe() throws CanalClientException {
|
||||
subscribe(""); // 传递空字符即可
|
||||
}
|
||||
|
||||
public void subscribe(String filter) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
currentConnector.subscribe(filter);
|
||||
this.filter = filter;
|
||||
return;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when subscribing from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
|
||||
throw new CanalClientException("failed to subscribe after " + times + " times retry.");
|
||||
}
|
||||
|
||||
public void unsubscribe() throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
currentConnector.unsubscribe();
|
||||
return;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
throw new CanalClientException("failed to unsubscribe after " + times + " times retry.");
|
||||
}
|
||||
|
||||
public Message get(int batchSize) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
Message msg = currentConnector.get(batchSize);
|
||||
return msg;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when getting data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||
}
|
||||
|
||||
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
Message msg = currentConnector.get(batchSize, timeout, unit);
|
||||
return msg;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when getting data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||
}
|
||||
|
||||
public Message getWithoutAck(int batchSize) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
Message msg = currentConnector.getWithoutAck(batchSize);
|
||||
return msg;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||
}
|
||||
|
||||
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
|
||||
return msg;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||
}
|
||||
|
||||
public void rollback(long batchId) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
currentConnector.rollback(batchId);
|
||||
return;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
throw new CanalClientException("failed to rollback after " + times + " times retry");
|
||||
}
|
||||
|
||||
public void rollback() throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
currentConnector.rollback();
|
||||
return;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
|
||||
throw new CanalClientException("failed to rollback after " + times + " times retry");
|
||||
}
|
||||
|
||||
public void ack(long batchId) throws CanalClientException {
|
||||
int times = 0;
|
||||
while (times < retryTimes) {
|
||||
try {
|
||||
currentConnector.ack(batchId);
|
||||
return;
|
||||
} catch (Throwable t) {
|
||||
logger.warn("something goes wrong when acking data from server:{}\n{}",
|
||||
currentConnector.getAddress(),
|
||||
ExceptionUtils.getFullStackTrace(t));
|
||||
times++;
|
||||
restart();
|
||||
logger.info("restart the connector for next round retry.");
|
||||
}
|
||||
}
|
||||
|
||||
throw new CanalClientException("failed to ack after " + times + " times retry");
|
||||
}
|
||||
|
||||
private void restart() throws CanalClientException {
|
||||
disconnect();
|
||||
try {
|
||||
Thread.sleep(retryInterval);
|
||||
} catch (InterruptedException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
connect();
|
||||
}
|
||||
|
||||
// ============================= setter / getter
|
||||
// ============================
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public int getSoTimeout() {
|
||||
return soTimeout;
|
||||
}
|
||||
|
||||
public void setSoTimeout(int soTimeout) {
|
||||
this.soTimeout = soTimeout;
|
||||
}
|
||||
|
||||
public int getRetryTimes() {
|
||||
return retryTimes;
|
||||
}
|
||||
|
||||
public void setRetryTimes(int retryTimes) {
|
||||
this.retryTimes = retryTimes;
|
||||
}
|
||||
|
||||
public int getRetryInterval() {
|
||||
return retryInterval;
|
||||
}
|
||||
|
||||
public void setRetryInterval(int retryInterval) {
|
||||
this.retryInterval = retryInterval;
|
||||
}
|
||||
|
||||
public CanalNodeAccessStrategy getAccessStrategy() {
|
||||
return accessStrategy;
|
||||
}
|
||||
|
||||
public void setAccessStrategy(CanalNodeAccessStrategy accessStrategy) {
|
||||
this.accessStrategy = accessStrategy;
|
||||
}
|
||||
|
||||
public SimpleCanalConnector getCurrentConnector() {
|
||||
return currentConnector;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package com.alibaba.otter.canal.client.impl;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.I0Itec.zkclient.IZkChildListener;
|
||||
import org.I0Itec.zkclient.IZkDataListener;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
|
||||
import com.alibaba.otter.canal.common.utils.JsonUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
|
||||
/**
|
||||
* 集群模式的调度策略
|
||||
*
|
||||
* @author jianghang 2012-12-3 下午10:01:04
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
|
||||
|
||||
private IZkChildListener childListener; // 监听所有的服务器列表
|
||||
private IZkDataListener dataListener; // 监听当前的工作节点
|
||||
private ZkClientx zkClient;
|
||||
private volatile List<InetSocketAddress> currentAddress = new ArrayList<InetSocketAddress>();
|
||||
private volatile InetSocketAddress runningAddress = null;
|
||||
|
||||
public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){
|
||||
this.zkClient = zkClient;
|
||||
childListener = new IZkChildListener() {
|
||||
|
||||
public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
|
||||
initClusters(currentChilds);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
dataListener = new IZkDataListener() {
|
||||
|
||||
public void handleDataDeleted(String dataPath) throws Exception {
|
||||
runningAddress = null;
|
||||
}
|
||||
|
||||
public void handleDataChange(String dataPath, Object data) throws Exception {
|
||||
initRunning(data);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
String clusterPath = ZookeeperPathUtils.getDestinationClusterRoot(destination);
|
||||
this.zkClient.subscribeChildChanges(clusterPath, childListener);
|
||||
initClusters(this.zkClient.getChildren(clusterPath));
|
||||
|
||||
String runningPath = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||
this.zkClient.subscribeDataChanges(runningPath, dataListener);
|
||||
initRunning(this.zkClient.readData(runningPath, true));
|
||||
}
|
||||
|
||||
public SocketAddress nextNode() {
|
||||
if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点
|
||||
return runningAddress;
|
||||
} else if (!currentAddress.isEmpty()) { // 如果不存在已经启动的服务，可能服务是一种lazy启动，随机选择一台触发服务器进行启动
|
||||
return currentAddress.get(0);// 默认返回第一个节点，之前已经做过shuffle
|
||||
} else {
|
||||
throw new CanalClientException("no alive canal server");
|
||||
}
|
||||
}
|
||||
|
||||
private void initClusters(List<String> currentChilds) {
|
||||
if (currentChilds == null || currentChilds.isEmpty()) {
|
||||
currentAddress = new ArrayList<InetSocketAddress>();
|
||||
} else {
|
||||
List<InetSocketAddress> addresses = new ArrayList<InetSocketAddress>();
|
||||
for (String address : currentChilds) {
|
||||
String[] strs = StringUtils.split(address, ":");
|
||||
if (strs != null && strs.length == 2) {
|
||||
addresses.add(new InetSocketAddress(strs[0], Integer.valueOf(strs[1])));
|
||||
}
|
||||
}
|
||||
|
||||
Collections.shuffle(addresses);
|
||||
currentAddress = addresses;// 直接切换引用
|
||||
}
|
||||
}
|
||||
|
||||
private void initRunning(Object data) {
|
||||
if (data == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
|
||||
String[] strs = StringUtils.split(runningData.getAddress(), ':');
|
||||
if (strs.length == 2) {
|
||||
runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
|
||||
}
|
||||
}
|
||||
|
||||
public void setZkClient(ZkClientx zkClient) {
|
||||
this.zkClient = zkClient;
|
||||
}
|
||||
|
||||
public ZkClientx getZkClient() {
|
||||
return zkClient;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,475 @@
|
||||
package com.alibaba.otter.canal.client.impl;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.channels.SocketChannel;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.impl.running.ClientRunningData;
|
||||
import com.alibaba.otter.canal.client.impl.running.ClientRunningListener;
|
||||
import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor;
|
||||
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||
import com.alibaba.otter.canal.common.utils.BooleanMutex;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Compression;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Get;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
|
||||
import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.InvalidProtocolBufferException;
|
||||
|
||||
/**
|
||||
* 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作
|
||||
*
|
||||
* @author jianghang 2012-10-24 下午05:37:20
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class SimpleCanalConnector implements CanalConnector {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SimpleCanalConnector.class);
|
||||
private SocketAddress address;
|
||||
private String username;
|
||||
private String password;
|
||||
private int soTimeout = 60000; // milliseconds
|
||||
private String filter; // 记录上一次的filter提交值,便于自动重试时提交
|
||||
|
||||
private final ByteBuffer readHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
|
||||
private final ByteBuffer writeHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
|
||||
private SocketChannel channel;
|
||||
private List<Compression> supportedCompressions = new ArrayList<Compression>();
|
||||
private ClientIdentity clientIdentity;
|
||||
private ClientRunningMonitor runningMonitor; // 运行控制
|
||||
private ZkClientx zkClientx;
|
||||
private BooleanMutex mutex = new BooleanMutex(false);
|
||||
private volatile boolean connected = false; // 代表connected是否已正常执行，因为有HA，不代表在工作中
|
||||
private boolean rollbackOnConnect = true; // 是否在connect链接成功后，自动执行rollback操作
|
||||
private boolean rollbackOnDisConnect = false; // 是否在connect链接成功后，自动执行rollback操作
|
||||
|
||||
// 读写数据分别使用不同的锁进行控制，减小锁粒度,读也需要排他锁，并发度容易造成数据包混乱，反序列化失败
|
||||
private Object readDataLock = new Object();
|
||||
private Object writeDataLock = new Object();
|
||||
|
||||
public SimpleCanalConnector(SocketAddress address, String username, String password, String destination){
|
||||
this(address, username, password, destination, 60000);
|
||||
}
|
||||
|
||||
public SimpleCanalConnector(SocketAddress address, String username, String password, String destination,
|
||||
int soTimeout){
|
||||
this.address = address;
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
this.soTimeout = soTimeout;
|
||||
this.clientIdentity = new ClientIdentity(destination, (short) 1001);
|
||||
}
|
||||
|
||||
public void connect() throws CanalClientException {
|
||||
if (connected) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (runningMonitor != null) {
|
||||
if (!runningMonitor.isStart()) {
|
||||
runningMonitor.start();
|
||||
}
|
||||
} else {
|
||||
waitClientRunning();
|
||||
doConnect();
|
||||
if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
|
||||
subscribe(filter);
|
||||
}
|
||||
if (rollbackOnConnect) {
|
||||
rollback();
|
||||
}
|
||||
}
|
||||
|
||||
connected = true;
|
||||
}
|
||||
|
||||
public void disconnect() throws CanalClientException {
|
||||
if (rollbackOnDisConnect && channel.isConnected()) {
|
||||
rollback();
|
||||
}
|
||||
|
||||
connected = false;
|
||||
if (runningMonitor != null) {
|
||||
if (runningMonitor.isStart()) {
|
||||
runningMonitor.stop();
|
||||
}
|
||||
} else {
|
||||
doDisconnnect();
|
||||
}
|
||||
}
|
||||
|
||||
private InetSocketAddress doConnect() throws CanalClientException {
|
||||
try {
|
||||
channel = SocketChannel.open();
|
||||
channel.socket().setSoTimeout(soTimeout);
|
||||
channel.connect(address);
|
||||
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||
if (p.getVersion() != 1) {
|
||||
throw new CanalClientException("unsupported version at this client.");
|
||||
}
|
||||
|
||||
if (p.getType() != PacketType.HANDSHAKE) {
|
||||
throw new CanalClientException("expect handshake but found other type.");
|
||||
}
|
||||
//
|
||||
Handshake handshake = Handshake.parseFrom(p.getBody());
|
||||
supportedCompressions.addAll(handshake.getSupportedCompressionsList());
|
||||
//
|
||||
ClientAuth ca = ClientAuth.newBuilder()
|
||||
.setUsername(username != null ? username : "")
|
||||
.setNetReadTimeout(soTimeout)
|
||||
.setNetWriteTimeout(soTimeout)
|
||||
.build();
|
||||
writeWithHeader(channel,
|
||||
Packet.newBuilder()
|
||||
.setType(PacketType.CLIENTAUTHENTICATION)
|
||||
.setBody(ca.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
//
|
||||
Packet ack = Packet.parseFrom(readNextPacket(channel));
|
||||
if (ack.getType() != PacketType.ACK) {
|
||||
throw new CanalClientException("unexpected packet type when ack is expected");
|
||||
}
|
||||
|
||||
Ack ackBody = Ack.parseFrom(ack.getBody());
|
||||
if (ackBody.getErrorCode() > 0) {
|
||||
throw new CanalClientException("something goes wrong when doing authentication: "
|
||||
+ ackBody.getErrorMessage());
|
||||
}
|
||||
|
||||
connected = true;
|
||||
return new InetSocketAddress(channel.socket().getLocalAddress(), channel.socket().getLocalPort());
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void doDisconnnect() throws CanalClientException {
|
||||
if (channel != null) {
|
||||
try {
|
||||
channel.close();
|
||||
} catch (IOException e) {
|
||||
logger.warn("exception on closing channel:{} \n {}", channel, e);
|
||||
}
|
||||
channel = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void subscribe() throws CanalClientException {
|
||||
subscribe(""); // 传递空字符即可
|
||||
}
|
||||
|
||||
public void subscribe(String filter) throws CanalClientException {
|
||||
waitClientRunning();
|
||||
try {
|
||||
writeWithHeader(channel,
|
||||
Packet.newBuilder()
|
||||
.setType(PacketType.SUBSCRIPTION)
|
||||
.setBody(Sub.newBuilder()
|
||||
.setDestination(clientIdentity.getDestination())
|
||||
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||
.setFilter(filter != null ? filter : "")
|
||||
.build()
|
||||
.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
//
|
||||
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||
Ack ack = Ack.parseFrom(p.getBody());
|
||||
if (ack.getErrorCode() > 0) {
|
||||
throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
|
||||
}
|
||||
|
||||
clientIdentity.setFilter(filter);
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void unsubscribe() throws CanalClientException {
|
||||
waitClientRunning();
|
||||
try {
|
||||
writeWithHeader(channel,
|
||||
Packet.newBuilder()
|
||||
.setType(PacketType.UNSUBSCRIPTION)
|
||||
.setBody(Unsub.newBuilder()
|
||||
.setDestination(clientIdentity.getDestination())
|
||||
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||
.build()
|
||||
.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
//
|
||||
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||
Ack ack = Ack.parseFrom(p.getBody());
|
||||
if (ack.getErrorCode() > 0) {
|
||||
throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public Message get(int batchSize) throws CanalClientException {
|
||||
return get(batchSize, null, null);
|
||||
}
|
||||
|
||||
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||
Message message = getWithoutAck(batchSize, timeout, unit);
|
||||
ack(message.getId());
|
||||
return message;
|
||||
}
|
||||
|
||||
public Message getWithoutAck(int batchSize) throws CanalClientException {
|
||||
return getWithoutAck(batchSize, null, null);
|
||||
}
|
||||
|
||||
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||
waitClientRunning();
|
||||
try {
|
||||
int size = (batchSize <= 0) ? 1000 : batchSize;
|
||||
long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制
|
||||
if (unit == null) {
|
||||
unit = TimeUnit.MILLISECONDS;
|
||||
}
|
||||
|
||||
writeWithHeader(channel,
|
||||
Packet.newBuilder()
|
||||
.setType(PacketType.GET)
|
||||
.setBody(Get.newBuilder()
|
||||
.setAutoAck(false)
|
||||
.setDestination(clientIdentity.getDestination())
|
||||
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||
.setFetchSize(size)
|
||||
.setTimeout(time)
|
||||
.setUnit(unit.ordinal())
|
||||
.build()
|
||||
.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
|
||||
return receiveMessages();
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private Message receiveMessages() throws InvalidProtocolBufferException, IOException {
|
||||
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||
switch (p.getType()) {
|
||||
case MESSAGES: {
|
||||
if (!p.getCompression().equals(Compression.NONE)) {
|
||||
throw new CanalClientException("compression is not supported in this connector");
|
||||
}
|
||||
|
||||
Messages messages = Messages.parseFrom(p.getBody());
|
||||
Message result = new Message(messages.getBatchId());
|
||||
for (ByteString byteString : messages.getMessagesList()) {
|
||||
result.addEntry(Entry.parseFrom(byteString));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case ACK: {
|
||||
Ack ack = Ack.parseFrom(p.getBody());
|
||||
throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
|
||||
}
|
||||
default: {
|
||||
throw new CanalClientException("unexpected packet type: " + p.getType());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void ack(long batchId) throws CanalClientException {
|
||||
waitClientRunning();
|
||||
ClientAck ca = ClientAck.newBuilder()
|
||||
.setDestination(clientIdentity.getDestination())
|
||||
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||
.setBatchId(batchId)
|
||||
.build();
|
||||
try {
|
||||
writeWithHeader(channel, Packet.newBuilder()
|
||||
.setType(PacketType.CLIENTACK)
|
||||
.setBody(ca.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void rollback(long batchId) throws CanalClientException {
|
||||
waitClientRunning();
|
||||
ClientRollback ca = ClientRollback.newBuilder()
|
||||
.setDestination(clientIdentity.getDestination())
|
||||
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||
.setBatchId(batchId)
|
||||
.build();
|
||||
try {
|
||||
writeWithHeader(channel, Packet.newBuilder()
|
||||
.setType(PacketType.CLIENTROLLBACK)
|
||||
.setBody(ca.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void rollback() throws CanalClientException {
|
||||
waitClientRunning();
|
||||
rollback(0);// 0代笔未设置
|
||||
}
|
||||
|
||||
// ==================== helper method ====================
|
||||
|
||||
private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
|
||||
synchronized (writeDataLock) {
|
||||
writeHeader.clear();
|
||||
writeHeader.putInt(body.length);
|
||||
writeHeader.flip();
|
||||
channel.write(writeHeader);
|
||||
channel.write(ByteBuffer.wrap(body));
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] readNextPacket(SocketChannel channel) throws IOException {
|
||||
synchronized (readDataLock) {
|
||||
readHeader.clear();
|
||||
read(channel, readHeader);
|
||||
int bodyLen = readHeader.getInt(0);
|
||||
ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen).order(ByteOrder.BIG_ENDIAN);
|
||||
read(channel, bodyBuf);
|
||||
return bodyBuf.array();
|
||||
}
|
||||
}
|
||||
|
||||
private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
|
||||
while (buffer.hasRemaining()) {
|
||||
int r = channel.read(buffer);
|
||||
if (r == -1) {
|
||||
throw new IOException("end of stream when reading header");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void initClientRunningMonitor(ClientIdentity clientIdentity) {
|
||||
if (zkClientx != null && clientIdentity != null && runningMonitor == null) {
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setClientId(clientIdentity.getClientId());
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
|
||||
runningMonitor = new ClientRunningMonitor();
|
||||
runningMonitor.setDestination(clientIdentity.getDestination());
|
||||
runningMonitor.setZkClient(zkClientx);
|
||||
runningMonitor.setClientData(clientData);
|
||||
runningMonitor.setListener(new ClientRunningListener() {
|
||||
|
||||
public InetSocketAddress processActiveEnter() {
|
||||
InetSocketAddress address = doConnect();
|
||||
mutex.set(true);
|
||||
if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
|
||||
subscribe(filter);
|
||||
}
|
||||
|
||||
if (rollbackOnConnect) {
|
||||
rollback();
|
||||
}
|
||||
|
||||
return address;
|
||||
}
|
||||
|
||||
public void processActiveExit() {
|
||||
mutex.set(false);
|
||||
doDisconnnect();
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void waitClientRunning() {
|
||||
try {
|
||||
if (zkClientx != null) {
|
||||
if (!connected) {// 未调用connect
|
||||
throw new CanalClientException("should connect first");
|
||||
}
|
||||
|
||||
mutex.get();// 阻塞等待
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean checkValid() {
|
||||
if (zkClientx != null) {
|
||||
return mutex.state();
|
||||
} else {
|
||||
return true;// 默认都放过
|
||||
}
|
||||
}
|
||||
|
||||
public SocketAddress getAddress() {
|
||||
return address;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public int getSoTimeout() {
|
||||
return soTimeout;
|
||||
}
|
||||
|
||||
public void setSoTimeout(int soTimeout) {
|
||||
this.soTimeout = soTimeout;
|
||||
}
|
||||
|
||||
public void setZkClientx(ZkClientx zkClientx) {
|
||||
this.zkClientx = zkClientx;
|
||||
initClientRunningMonitor(this.clientIdentity);
|
||||
}
|
||||
|
||||
public void setRollbackOnConnect(boolean rollbackOnConnect) {
|
||||
this.rollbackOnConnect = rollbackOnConnect;
|
||||
}
|
||||
|
||||
public void setRollbackOnDisConnect(boolean rollbackOnDisConnect) {
|
||||
this.rollbackOnDisConnect = rollbackOnDisConnect;
|
||||
}
|
||||
|
||||
public void setFilter(String filter) {
|
||||
this.filter = filter;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.alibaba.otter.canal.client.impl;
|
||||
|
||||
import java.net.SocketAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
|
||||
|
||||
/**
|
||||
* 简单版本的node访问实现
|
||||
*
|
||||
* @author jianghang 2012-10-29 下午08:00:23
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
|
||||
|
||||
private List<SocketAddress> nodes = new ArrayList<SocketAddress>();
|
||||
private int index = 0;
|
||||
|
||||
public SimpleNodeAccessStrategy(List<? extends SocketAddress> nodes){
|
||||
if (nodes == null || nodes.size() < 1) {
|
||||
throw new IllegalArgumentException("at least 1 node required.");
|
||||
}
|
||||
this.nodes.addAll(nodes);
|
||||
}
|
||||
|
||||
public SocketAddress nextNode() {
|
||||
try {
|
||||
return nodes.get(index);
|
||||
} finally {
|
||||
index = (index + 1) % nodes.size();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.alibaba.otter.canal.client.impl.running;
|
||||
|
||||
/**
|
||||
* client running状态信息
|
||||
*
|
||||
* @author jianghang 2012-11-22 下午03:41:50
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ClientRunningData {
|
||||
|
||||
private short clientId;
|
||||
private String address;
|
||||
private boolean active = true;
|
||||
|
||||
public short getClientId() {
|
||||
return clientId;
|
||||
}
|
||||
|
||||
public void setClientId(short clientId) {
|
||||
this.clientId = clientId;
|
||||
}
|
||||
|
||||
public String getAddress() {
|
||||
return address;
|
||||
}
|
||||
|
||||
public void setAddress(String address) {
|
||||
this.address = address;
|
||||
}
|
||||
|
||||
public boolean isActive() {
|
||||
return active;
|
||||
}
|
||||
|
||||
public void setActive(boolean active) {
|
||||
this.active = active;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.alibaba.otter.canal.client.impl.running;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
/**
|
||||
* 触发一下mainstem发生切换
|
||||
*
|
||||
* @author jianghang 2012-9-11 下午02:26:03
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface ClientRunningListener {
|
||||
|
||||
/**
|
||||
* 触发现在轮到自己做为active，需要载入上一个active的上下文数据
|
||||
*/
|
||||
public InetSocketAddress processActiveEnter();
|
||||
|
||||
/**
|
||||
* 触发一下当前active模式失败
|
||||
*/
|
||||
public void processActiveExit();
|
||||
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package com.alibaba.otter.canal.client.impl.running;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.I0Itec.zkclient.IZkDataListener;
|
||||
import org.I0Itec.zkclient.exception.ZkException;
|
||||
import org.I0Itec.zkclient.exception.ZkInterruptedException;
|
||||
import org.I0Itec.zkclient.exception.ZkNoNodeException;
|
||||
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.utils.BooleanMutex;
|
||||
import com.alibaba.otter.canal.common.utils.JsonUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||
|
||||
/**
|
||||
* clinet running控制
|
||||
*
|
||||
* @author jianghang 2012-11-22 下午03:43:01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ClientRunningMonitor extends AbstractCanalLifeCycle {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class);
|
||||
private ZkClientx zkClient;
|
||||
private String destination;
|
||||
private ClientRunningData clientData;
|
||||
private IZkDataListener dataListener;
|
||||
private BooleanMutex mutex = new BooleanMutex(false);
|
||||
private volatile boolean release = false;
|
||||
private volatile ClientRunningData activeData;
|
||||
private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1);
|
||||
private ClientRunningListener listener;
|
||||
private int delayTime = 5;
|
||||
|
||||
public ClientRunningMonitor(){
|
||||
dataListener = new IZkDataListener() {
|
||||
|
||||
public void handleDataChange(String dataPath, Object data) throws Exception {
|
||||
MDC.put("destination", destination);
|
||||
ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class);
|
||||
if (!isMine(runningData.getAddress())) {
|
||||
mutex.set(false);
|
||||
}
|
||||
|
||||
if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
|
||||
release = true;
|
||||
releaseRunning();// 彻底释放mainstem
|
||||
}
|
||||
|
||||
activeData = (ClientRunningData) runningData;
|
||||
}
|
||||
|
||||
public void handleDataDeleted(String dataPath) throws Exception {
|
||||
MDC.put("destination", destination);
|
||||
mutex.set(false);
|
||||
if (!release && activeData != null && isMine(activeData.getAddress())) {
|
||||
// 如果上一次active的状态就是本机，则即时触发一下active抢占
|
||||
initRunning();
|
||||
} else {
|
||||
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
|
||||
delayExector.schedule(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
initRunning();
|
||||
}
|
||||
}, delayTime, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
public void start() {
|
||||
super.start();
|
||||
|
||||
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||
zkClient.subscribeDataChanges(path, dataListener);
|
||||
initRunning();
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
super.stop();
|
||||
|
||||
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||
zkClient.unsubscribeDataChanges(path, dataListener);
|
||||
releaseRunning(); // 尝试一下release
|
||||
}
|
||||
|
||||
public void initRunning() {
|
||||
if (!isStart()) {
|
||||
return;
|
||||
}
|
||||
|
||||
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||
// 序列化
|
||||
byte[] bytes = JsonUtils.marshalToByte(clientData);
|
||||
try {
|
||||
mutex.set(false);
|
||||
zkClient.create(path, bytes, CreateMode.EPHEMERAL);
|
||||
processActiveEnter();// 触发一下事件
|
||||
activeData = clientData;
|
||||
mutex.set(true);
|
||||
} catch (ZkNodeExistsException e) {
|
||||
bytes = zkClient.readData(path, true);
|
||||
if (bytes == null) {// 如果不存在节点，立即尝试一次
|
||||
initRunning();
|
||||
} else {
|
||||
activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
|
||||
}
|
||||
} catch (ZkNoNodeException e) {
|
||||
zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()),
|
||||
true); // 尝试创建父节点
|
||||
initRunning();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 阻塞等待自己成为active，如果自己成为active，立马返回
|
||||
*
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void waitForActive() throws InterruptedException {
|
||||
initRunning();
|
||||
mutex.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查当前的状态
|
||||
*/
|
||||
public boolean check() {
|
||||
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||
try {
|
||||
byte[] bytes = zkClient.readData(path);
|
||||
ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
|
||||
activeData = eventData;// 更新下为最新值
|
||||
// 检查下nid是否为自己
|
||||
boolean result = isMine(activeData.getAddress());
|
||||
if (!result) {
|
||||
logger.warn("canal is running in [{}] , but not in [{}]",
|
||||
activeData.getAddress(),
|
||||
clientData.getAddress());
|
||||
}
|
||||
return result;
|
||||
} catch (ZkNoNodeException e) {
|
||||
logger.warn("canal is not run any in node");
|
||||
return false;
|
||||
} catch (ZkInterruptedException e) {
|
||||
logger.warn("canal check is interrupt");
|
||||
Thread.interrupted();// 清除interrupt标记
|
||||
return check();
|
||||
} catch (ZkException e) {
|
||||
logger.warn("canal check is failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean releaseRunning() {
|
||||
if (check()) {
|
||||
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||
zkClient.delete(path);
|
||||
mutex.set(false);
|
||||
processActiveExit();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ====================== helper method ======================
|
||||
|
||||
private boolean isMine(String address) {
|
||||
return address.equals(clientData.getAddress());
|
||||
}
|
||||
|
||||
private void processActiveEnter() {
|
||||
if (listener != null) {
|
||||
// 触发回调，建立与server的socket链接
|
||||
InetSocketAddress connectAddress = listener.processActiveEnter();
|
||||
String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
|
||||
this.clientData.setAddress(address);
|
||||
|
||||
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
|
||||
this.clientData.getClientId());
|
||||
// 序列化
|
||||
byte[] bytes = JsonUtils.marshalToByte(clientData);
|
||||
zkClient.writeData(path, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
private void processActiveExit() {
|
||||
if (listener != null) {
|
||||
listener.processActiveExit();
|
||||
}
|
||||
}
|
||||
|
||||
public void setListener(ClientRunningListener listener) {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
// ===================== setter / getter =======================
|
||||
|
||||
public void setDestination(String destination) {
|
||||
this.destination = destination;
|
||||
}
|
||||
|
||||
public void setClientData(ClientRunningData clientData) {
|
||||
this.clientData = clientData;
|
||||
}
|
||||
|
||||
public void setDelayTime(int delayTime) {
|
||||
this.delayTime = delayTime;
|
||||
}
|
||||
|
||||
public void setZkClient(ZkClientx zkClient) {
|
||||
this.zkClient = zkClient;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.alibaba.otter.canal.client.running;
|
||||
|
||||
import org.junit.Assert;
|
||||
|
||||
public class AbstractZkTest {
|
||||
|
||||
protected String destination = "ljhtest1";
|
||||
protected String cluster1 = "127.0.0.1:2188";
|
||||
protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188";
|
||||
|
||||
public void sleep(long time) {
|
||||
try {
|
||||
Thread.sleep(time);
|
||||
} catch (InterruptedException e) {
|
||||
Assert.fail(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package com.alibaba.otter.canal.client.running;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.alibaba.otter.canal.client.impl.running.ClientRunningData;
|
||||
import com.alibaba.otter.canal.client.impl.running.ClientRunningListener;
|
||||
import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor;
|
||||
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||
|
||||
public class ClientRunningTest extends AbstractZkTest {
|
||||
|
||||
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||
private short clientId = 1001;
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
|
||||
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOneServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
|
||||
runningMonitor.start();
|
||||
sleep(2000L);
|
||||
runningMonitor.stop();
|
||||
sleep(2000L);
|
||||
|
||||
if (countLatch.getCount() != 0) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
|
||||
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
|
||||
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
|
||||
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor1.isStart()) {
|
||||
runningMonitor1.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor1.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor2.isStart()) {
|
||||
runningMonitor2.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor2.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor3.isStart()) {
|
||||
runningMonitor3.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor3.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
sleep(30000L);
|
||||
}
|
||||
|
||||
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
|
||||
final int port) {
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setClientId(clientId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
|
||||
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
|
||||
runningMonitor.setDestination(destination);
|
||||
runningMonitor.setZkClient(zkclientx);
|
||||
runningMonitor.setClientData(clientData);
|
||||
runningMonitor.setListener(new ClientRunningListener() {
|
||||
|
||||
public InetSocketAddress processActiveEnter() {
|
||||
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
|
||||
countLatch.countDown();
|
||||
return new InetSocketAddress(AddressUtils.getHostIp(), port);
|
||||
}
|
||||
|
||||
public void processActiveExit() {
|
||||
countLatch.countDown();
|
||||
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
|
||||
}
|
||||
|
||||
});
|
||||
runningMonitor.setDelayTime(1);
|
||||
return runningMonitor;
|
||||
}
|
||||
}
|
||||
14
client/src/test/java/logback.xml
Normal file
14
client/src/test/java/logback.xml
Normal file
@@ -0,0 +1,14 @@
|
||||
<configuration scan="true" scanPeriod=" 5 seconds">
|
||||
|
||||
<jmxConfigurator />
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<root level="WARN">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
</configuration>
|
||||
Reference in New Issue
Block a user