chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
server/src/main/java/com/alibaba/otter/canal/server/CanalServer.java Normal file
View File

@@ -0,0 +1,17 @@
package com.alibaba.otter.canal.server;
import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.alibaba.otter.canal.server.exception.CanalServerException;
/**
* 对应canal整个服务实例，一个jvm实例只有一份server
*
* @author jianghang 2012-7-12 下午01:32:29
* @version 1.0.0
*/
public interface CanalServer extends CanalLifeCycle {
public void start() throws CanalServerException;
public void stop() throws CanalServerException;
}

459
server/src/main/java/com/alibaba/otter/canal/server/embeded/CanalServerWithEmbeded.java Normal file
View File

@@ -0,0 +1,459 @@
package com.alibaba.otter.canal.server.embeded;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.alibaba.otter.canal.server.CanalServer;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.store.CanalEventStore;
import com.alibaba.otter.canal.store.model.Event;
import com.alibaba.otter.canal.store.model.Events;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
/**
* 嵌入式版本实现
*
* @author jianghang 2012-7-12 下午01:34:00
* @author zebin.xuzb
* @version 1.0.0
*/
public class CanalServerWithEmbeded extends AbstractCanalLifeCycle implements CanalServer {
private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbeded.class);
private Map<String, CanalInstance> canalInstances;
// private Map<ClientIdentity, Position> lastRollbackPostions;
private CanalInstanceGenerator canalInstanceGenerator;
public void start() {
super.start();
canalInstances = new MapMaker().makeComputingMap(new Function<String, CanalInstance>() {
public CanalInstance apply(String destination) {
CanalInstance canalInstance = canalInstanceGenerator.generate(destination);
return canalInstance;
}
});
// lastRollbackPostions = new MapMaker().makeMap();
}
public void stop() {
super.stop();
for (Map.Entry<String, CanalInstance> entry : canalInstances.entrySet()) {
try {
CanalInstance instance = entry.getValue();
if (instance.isStart()) {
try {
String destination = entry.getKey();
MDC.put("destination", destination);
entry.getValue().stop();
logger.info("stop CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
}
}
} catch (Exception e) {
logger.error(String.format("stop cannalInstance[%s] has an error", entry.getKey()), e);
}
}
}
public void start(final String destination) {
final CanalInstance canalInstance = canalInstances.get(destination);
if (!canalInstance.isStart()) {
try {
MDC.put("destination", destination);
canalInstance.start();
logger.info("start CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
}
}
}
public void stop(String destination) {
CanalInstance canalInstance = canalInstances.remove(destination);
if (canalInstance != null) {
if (canalInstance.isStart()) {
try {
MDC.put("destination", destination);
canalInstance.stop();
logger.info("stop CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
}
}
}
}
public boolean isStart(String destination) {
return canalInstances.containsKey(destination) && canalInstances.get(destination).isStart();
}
/**
* 客户端订阅，重复订阅时会更新对应的filter信息
*/
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
if (!canalInstance.getMetaManager().isStart()) {
canalInstance.getMetaManager().start();
}
canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅
Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
if (position == null) {
position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
if (position != null) {
canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
}
logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
} else {
logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
}
// 通知下订阅关系变化
canalInstance.subscribeChange(clientIdentity);
}
/**
* 取消订阅
*/
public void unsubscribe(ClientIdentity clientIdentity) throws CanalServerException {
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
canalInstance.getMetaManager().unsubscribe(clientIdentity); // 执行一下meta订阅
logger.info("unsubscribe successfully, {}", clientIdentity);
}
/**
* 查询所有的订阅信息
*/
public List<ClientIdentity> listAllSubscribe(String destination) throws CanalServerException {
CanalInstance canalInstance = canalInstances.get(destination);
return canalInstance.getMetaManager().listAllSubscribeInfo(destination);
}
/**
* 获取数据
*
* <pre>
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
* </pre>
*/
public Message get(ClientIdentity clientIdentity, int batchSize) throws CanalServerException {
return get(clientIdentity, batchSize, null, null);
}
/**
* 获取数据，可以指定超时时间.
*
* <pre>
* 几种case:
* a. 如果timeout为null，则采用tryGet方式，即时获取
* b. 如果timeout不为null
* 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
* 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
*
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
* </pre>
*/
public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
synchronized (canalInstance) {
// 获取到流式数据中的最后一批获取的位置
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
if (positionRanges != null) {
throw new CanalServerException(
String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data",
clientIdentity.getClientId(), positionRanges));
}
Events<Event> events = null;
Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
if (CollectionUtils.isEmpty(events.getEvents())) {
logger.debug("get successfully, clientId:{} batchSize:{} but result is null", new Object[] {
clientIdentity.getClientId(), batchSize });
return new Message(-1, new ArrayList<Entry>()); // 返回空包，避免生成batchId，浪费性能
} else {
// 记录到流式信息
Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
List<Entry> entrys = Lists.transform(events.getEvents(), new Function<Event, Entry>() {
public Entry apply(Event input) {
return input.getEntry();
}
});
logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
new Object[] { clientIdentity.getClientId(), batchSize, entrys.size(), batchId,
events.getPositionRange() });
// 直接提交ack
ack(clientIdentity, batchId);
return new Message(batchId, entrys);
}
}
}
/**
* 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
*
* <pre>
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
* </pre>
*/
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize) throws CanalServerException {
return getWithoutAck(clientIdentity, batchSize, null, null);
}
/**
* 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
*
* <pre>
* 几种case:
* a. 如果timeout为null，则采用tryGet方式，即时获取
* b. 如果timeout不为null
* 1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
* 2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
*
* 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
* </pre>
*/
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
synchronized (canalInstance) {
// 获取到流式数据中的最后一批获取的位置
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
Events<Event> events = null;
if (positionRanges != null) { // 存在流数据
events = getEvents(canalInstance.getEventStore(), positionRanges.getStart(), batchSize, timeout, unit);
} else {// ack后第一次获取
Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
if (start == null) { // 第一次，还没有过ack记录，则获取当前store中的第一条
start = canalInstance.getEventStore().getFirstPosition();
}
events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
}
if (CollectionUtils.isEmpty(events.getEvents())) {
logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result is null", new Object[] {
clientIdentity.getClientId(), batchSize });
return new Message(-1, new ArrayList<Entry>()); // 返回空包，避免生成batchId，浪费性能
} else {
// 记录到流式信息
Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
List<Entry> entrys = Lists.transform(events.getEvents(), new Function<Event, Entry>() {
public Entry apply(Event input) {
return input.getEntry();
}
});
logger.info("getWithoutAck successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
new Object[] { clientIdentity.getClientId(), batchSize, entrys.size(), batchId,
events.getPositionRange() });
return new Message(batchId, entrys);
}
}
}
/**
* 查询当前未被ack的batch列表，batchId会按照从小到大进行返回
*/
public List<Long> listBatchIds(ClientIdentity clientIdentity) throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
Map<Long, PositionRange> batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity);
List<Long> result = new ArrayList<Long>(batchs.keySet());
Collections.sort(result);
return result;
}
/**
* 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
*
* <pre>
* 注意：进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
* </pre>
*/
public void ack(ClientIdentity clientIdentity, long batchId) throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
PositionRange<LogPosition> positionRanges = null;
positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId); // 更新位置
if (positionRanges == null) { // 说明是重复的ack/rollback
throw new CanalServerException(
String.format("ack error , clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(), batchId));
}
// 更新cursor最好严格判断下位置是否有跳跃更新
// Position position = lastRollbackPostions.get(clientIdentity);
// if (position != null) {
// // Position position =
// canalInstance.getMetaManager().getCursor(clientIdentity);
// LogPosition minPosition =
// CanalEventUtils.min(positionRanges.getStart(), (LogPosition)
// position);
// if (minPosition == position) {// ack的position要晚于该最后ack的位置，可能有丢数据
// throw new CanalServerException(
// String.format(
// "ack error , clientId:%s batchId:%d %s is jump ack , last ack:%s",
// clientIdentity.getClientId(), batchId, positionRanges,
// position));
// }
// }
// 更新cursor
if (positionRanges.getAck() != null) {
canalInstance.getMetaManager().updateCursor(clientIdentity, positionRanges.getAck());
logger.info("ack successfully, clientId:{} batchId:{} position:{}",
new Object[] { clientIdentity.getClientId(), batchId, positionRanges });
}
// 可定时清理数据
canalInstance.getEventStore().ack(positionRanges.getEnd());
}
/**
* 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
*/
public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
checkStart(clientIdentity.getDestination());
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
// 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
if (!hasSubscribe) {
return;
}
synchronized (canalInstance) {
// 清除batch信息
canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
// rollback eventStore中的状态信息
canalInstance.getEventStore().rollback();
logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
}
}
/**
* 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
*/
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
checkStart(clientIdentity.getDestination());
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
// 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
if (!hasSubscribe) {
return;
}
synchronized (canalInstance) {
// 清除batch信息
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
batchId);
if (positionRanges == null) { // 说明是重复的ack/rollback
throw new CanalServerException(
String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(), batchId));
}
// lastRollbackPostions.put(clientIdentity,
// positionRanges.getEnd());// 记录一下最后rollback的位置
// TODO 后续rollback到指定的batchId位置
canalInstance.getEventStore().rollback();// rollback
// eventStore中的状态信息
logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
new Object[] { clientIdentity.getClientId(), batchId, positionRanges });
}
}
public Map<String, CanalInstance> getCanalInstances() {
return Maps.newHashMap(canalInstances);
}
// ======================== helper method =======================
/**
* 根据不同的参数，选择不同的方式获取数据
*/
private Events<Event> getEvents(CanalEventStore eventStore, Position start, int batchSize, Long timeout,
TimeUnit unit) {
if (timeout == null) {
return eventStore.tryGet(start, batchSize);
} else {
try {
if (timeout <= 0) {
return eventStore.get(start, batchSize);
} else {
return eventStore.get(start, batchSize, timeout, unit);
}
} catch (Exception e) {
throw new CanalServerException(e);
}
}
}
private void checkSubscribe(ClientIdentity clientIdentity) {
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
if (!hasSubscribe) {
throw new CanalServerException(String.format("ClientIdentity:%s should subscribe first",
clientIdentity.toString()));
}
}
private void checkStart(String destination) {
if (!isStart(destination)) {
throw new CanalServerException(String.format("destination:%s should start first", destination));
}
}
// ========= setter ==========
public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) {
this.canalInstanceGenerator = canalInstanceGenerator;
}
}

35
server/src/main/java/com/alibaba/otter/canal/server/exception/CanalServerException.java Normal file
View File

@@ -0,0 +1,35 @@
package com.alibaba.otter.canal.server.exception;
import com.alibaba.otter.canal.common.CanalException;
/**
* canal 异常定义
*
* @author jianghang 2012-6-15 下午04:57:35
* @version 1.0.0
*/
public class CanalServerException extends CanalException {
private static final long serialVersionUID = -7288830284122672209L;
public CanalServerException(String errorCode){
super(errorCode);
}
public CanalServerException(String errorCode, Throwable cause){
super(errorCode, cause);
}
public CanalServerException(String errorCode, String errorDesc){
super(errorCode + ":" + errorDesc);
}
public CanalServerException(String errorCode, String errorDesc, Throwable cause){
super(errorCode + ":" + errorDesc, cause);
}
public CanalServerException(Throwable cause){
super(cause);
}
}

105
server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNetty.java Normal file
View File

@@ -0,0 +1,105 @@
package com.alibaba.otter.canal.server.netty;
import java.net.InetSocketAddress;
import java.util.concurrent.Executors;
import org.apache.commons.lang.StringUtils;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.server.CanalServer;
import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
import com.alibaba.otter.canal.server.netty.handler.ClientAuthenticationHandler;
import com.alibaba.otter.canal.server.netty.handler.FixedHeaderFrameDecoder;
import com.alibaba.otter.canal.server.netty.handler.HandshakeInitializationHandler;
import com.alibaba.otter.canal.server.netty.handler.SessionHandler;
/**
* 基于netty网络服务的server实现
*
* @author jianghang 2012-7-12 下午01:34:49
* @version 1.0.0
*/
public class CanalServerWithNetty extends AbstractCanalLifeCycle implements CanalServer {
private CanalServerWithEmbeded embededServer; // 嵌入式server
private String ip;
private int port;
private Channel serverChannel = null;
private ServerBootstrap bootstrap = null;
public CanalServerWithNetty(){
}
public CanalServerWithNetty(CanalServerWithEmbeded embededServer){
this.embededServer = embededServer;
}
public void start() {
super.start();
if (!embededServer.isStart()) {
embededServer.start();
}
this.bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),
Executors.newCachedThreadPool()));
// 构造对应的pipeline
bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipelines = Channels.pipeline();
pipelines.addLast(FixedHeaderFrameDecoder.class.getName(), new FixedHeaderFrameDecoder());
pipelines.addLast(HandshakeInitializationHandler.class.getName(), new HandshakeInitializationHandler());
pipelines.addLast(ClientAuthenticationHandler.class.getName(),
new ClientAuthenticationHandler(embededServer));
SessionHandler sessionHandler = new SessionHandler(embededServer);
pipelines.addLast(SessionHandler.class.getName(), sessionHandler);
return pipelines;
}
});
// 启动
if (StringUtils.isNotEmpty(ip)) {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.ip, this.port));
} else {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.port));
}
}
public void stop() {
super.stop();
if (this.serverChannel != null) {
this.serverChannel.close().awaitUninterruptibly(1000);
}
if (this.bootstrap != null) {
this.bootstrap.releaseExternalResources();
}
if (embededServer.isStart()) {
embededServer.stop();
}
}
public void setIp(String ip) {
this.ip = ip;
}
public void setPort(int port) {
this.port = port;
}
public void setEmbededServer(CanalServerWithEmbeded embededServer) {
this.embededServer = embededServer;
}
}

55
server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java Normal file
View File

@@ -0,0 +1,55 @@
package com.alibaba.otter.canal.server.netty;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.util.HashedWheelTimer;
import org.jboss.netty.util.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
public class NettyUtils {
private static final Logger logger = LoggerFactory.getLogger(NettyUtils.class);
private static int HEADER_LENGTH = 4;
public static Timer hashedWheelTimer = new HashedWheelTimer();
public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
if (channelFutureListner == null) {
Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body));
} else {
Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body)).addListener(channelFutureListner);
}
}
public static void ack(Channel channel, ChannelFutureListener channelFutureListner) {
write(
channel,
Packet.newBuilder().setType(CanalPacket.PacketType.ACK).setBody(Ack.newBuilder().build().toByteString()).build().toByteArray(),
channelFutureListner);
}
public static void error(int errorCode, String errorMessage, Channel channel,
ChannelFutureListener channelFutureListener) {
if (channelFutureListener == null) {
channelFutureListener = ChannelFutureListener.CLOSE;
}
logger.error("ErrotCode:{} , Caused by : \n{}", errorCode, errorMessage);
write(
channel,
Packet.newBuilder().setType(CanalPacket.PacketType.ACK).setBody(
Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(
errorMessage).build().toByteString()).build().toByteArray(),
channelFutureListener);
}
}

121
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java Normal file
View File

@@ -0,0 +1,121 @@
package com.alibaba.otter.canal.server.netty.handler;
import org.apache.commons.lang.StringUtils;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateEvent;
import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
import com.alibaba.otter.canal.server.netty.NettyUtils;
/**
* 客户端身份认证处理
*
* @author jianghang 2012-10-24 上午11:12:45
* @version 1.0.0
*/
public class ClientAuthenticationHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(ClientAuthenticationHandler.class);
private final int SUPPORTED_VERSION = 3;
private final int defaultSubscriptorDisconnectIdleTimeout = 5 * 60 * 1000;
private CanalServerWithEmbeded embededServer;
public ClientAuthenticationHandler(){
}
public ClientAuthenticationHandler(CanalServerWithEmbeded embededServer){
this.embededServer = embededServer;
}
public void messageReceived(final ChannelHandlerContext ctx, MessageEvent e) throws Exception {
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
final Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
switch (packet.getVersion()) {
case SUPPORTED_VERSION:
default:
final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
// 如果存在订阅信息
if (StringUtils.isNotEmpty(clientAuth.getDestination())
&& StringUtils.isNotEmpty(clientAuth.getClientId())) {
ClientIdentity clientIdentity = new ClientIdentity(clientAuth.getDestination(),
Short.valueOf(clientAuth.getClientId()),
clientAuth.getFilter());
try {
MDC.put("destination", clientIdentity.getDestination());
embededServer.subscribe(clientIdentity);
ctx.setAttachment(clientIdentity);// 设置状态数据
// 尝试启动，如果已经启动，忽略
if (!embededServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (!runningMonitor.isStart()) {
runningMonitor.start();
}
}
} finally {
MDC.remove("destination");
}
}
NettyUtils.ack(ctx.getChannel(), new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
logger.info("remove unused channel handlers after authentication is done successfully.");
ctx.getPipeline().remove(HandshakeInitializationHandler.class.getName());
ctx.getPipeline().remove(ClientAuthenticationHandler.class.getName());
int readTimeout = defaultSubscriptorDisconnectIdleTimeout;
int writeTimeout = defaultSubscriptorDisconnectIdleTimeout;
if (clientAuth.getNetReadTimeout() > 0) {
readTimeout = clientAuth.getNetReadTimeout();
}
if (clientAuth.getNetWriteTimeout() > 0) {
writeTimeout = clientAuth.getNetWriteTimeout();
}
IdleStateHandler idleStateHandler = new IdleStateHandler(NettyUtils.hashedWheelTimer,
readTimeout,
writeTimeout,
0);
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateHandler.class.getName(),
idleStateHandler);
IdleStateAwareChannelHandler idleStateAwareChannelHandler = new IdleStateAwareChannelHandler() {
public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) throws Exception {
logger.warn("channel:{} idle timeout exceeds, close channel to save server resources...",
ctx.getChannel());
ctx.getChannel().close();
}
};
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateAwareChannelHandler.class.getName(),
idleStateAwareChannelHandler);
}
});
break;
}
}
public void setEmbededServer(CanalServerWithEmbeded embededServer) {
this.embededServer = embededServer;
}
}

21
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/FixedHeaderFrameDecoder.java Normal file
View File

@@ -0,0 +1,21 @@
package com.alibaba.otter.canal.server.netty.handler;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.replay.ReplayingDecoder;
import org.jboss.netty.handler.codec.replay.VoidEnum;
/**
* 解析对应的header信息
*
* @author jianghang 2012-10-24 上午11:31:39
* @version 1.0.0
*/
public class FixedHeaderFrameDecoder extends ReplayingDecoder<VoidEnum> {
protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer, VoidEnum state)
throws Exception {
return buffer.readBytes(buffer.readInt());
}
}

30
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java Normal file
View File

@@ -0,0 +1,30 @@
package com.alibaba.otter.canal.server.netty.handler;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
import com.alibaba.otter.canal.server.netty.NettyUtils;
/**
* handshake交互
*
* @author jianghang 2012-10-24 上午11:39:54
* @version 1.0.0
*/
public class HandshakeInitializationHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(HandshakeInitializationHandler.class);
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
byte[] body = Packet.newBuilder().setType(CanalPacket.PacketType.HANDSHAKE).setBody(
Handshake.newBuilder().build().toByteString()).build().toByteArray();
NettyUtils.write(ctx.getChannel(), body, null);
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}
}

249
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java Normal file
View File

@@ -0,0 +1,249 @@
package com.alibaba.otter.canal.server.netty.handler;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.slf4j.helpers.MessageFormatter;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
import com.alibaba.otter.canal.protocol.CanalPacket.Get;
import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
import com.alibaba.otter.canal.server.netty.NettyUtils;
/**
* 处理具体的客户端请求
*
* @author jianghang 2012-10-24 下午02:21:13
* @version 1.0.0
*/
public class SessionHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class);
private CanalServerWithEmbeded embededServer;
public SessionHandler(){
}
public SessionHandler(CanalServerWithEmbeded embededServer){
this.embededServer = embededServer;
}
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
logger.info("message receives in session handler...");
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
ClientIdentity clientIdentity = null;
try {
switch (packet.getType()) {
case SUBSCRIPTION:
Sub sub = Sub.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()),
sub.getFilter());
MDC.put("destination", clientIdentity.getDestination());
embededServer.subscribe(clientIdentity);
// 尝试启动，如果已经启动，忽略
if (!embededServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (!runningMonitor.isStart()) {
runningMonitor.start();
}
}
ctx.setAttachment(clientIdentity);// 设置状态数据
NettyUtils.ack(ctx.getChannel(), null);
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage(),
ctx.getChannel(), null);
}
break;
case UNSUBSCRIPTION:
Unsub unsub = Unsub.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()),
unsub.getFilter());
MDC.put("destination", clientIdentity.getDestination());
embededServer.unsubscribe(clientIdentity);
stopCanalInstanceIfNecessary(clientIdentity);// 尝试关闭
NettyUtils.ack(ctx.getChannel(), null);
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage(),
ctx.getChannel(), null);
}
break;
case GET:
Get get = CanalPacket.Get.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
MDC.put("destination", clientIdentity.getDestination());
Message message = null;
// if (get.getAutoAck()) {
// if (get.getTimeout() == -1) {//是否是初始值
// message = embededServer.get(clientIdentity, get.getFetchSize());
// } else {
// TimeUnit unit = convertTimeUnit(get.getUnit());
// message = embededServer.get(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
// }
// } else {
if (get.getTimeout() == -1) {//是否是初始值
message = embededServer.getWithoutAck(clientIdentity, get.getFetchSize());
} else {
TimeUnit unit = convertTimeUnit(get.getUnit());
message = embededServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(),
unit);
}
// }
Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
messageBuilder.setBatchId(message.getId());
if (message.getId() != -1 && !CollectionUtils.isEmpty(message.getEntries())) {
for (Entry entry : message.getEntries()) {
messageBuilder.addMessages(entry.toByteString());
}
}
packetBuilder.setBody(messageBuilder.build().toByteString());
NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(),
ctx.getChannel(), null);
}
break;
case CLIENTACK:
ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
MDC.put("destination", ack.getDestination());
if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
if (ack.getBatchId() == 0L) {
NettyUtils.error(402,
MessageFormatter.format("batchId should assign value", ack.toString()).getMessage(),
ctx.getChannel(), null);
} else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据，直接忽略之
// donothing
} else {
clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
embededServer.ack(clientIdentity, ack.getBatchId());
}
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage(),
ctx.getChannel(), null);
}
break;
case CLIENTROLLBACK:
ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
MDC.put("destination", rollback.getDestination());
if (StringUtils.isNotEmpty(rollback.getDestination())
&& StringUtils.isNotEmpty(rollback.getClientId())) {
clientIdentity = new ClientIdentity(rollback.getDestination(),
Short.valueOf(rollback.getClientId()));
if (rollback.getBatchId() == 0L) {
embededServer.rollback(clientIdentity);// 回滚所有批次
} else {
embededServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次
}
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage(),
ctx.getChannel(), null);
}
break;
default:
NettyUtils.error(400,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage(),
ctx.getChannel(), null);
break;
}
} catch (Throwable exception) {
NettyUtils.error(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage(),
ctx.getChannel(), null);
} finally {
MDC.remove("destination");
}
}
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
logger.error("something goes wrong with channel:{}, exception={}", ctx.getChannel(),
ExceptionUtils.getStackTrace(e.getCause()));
ctx.getChannel().close();
}
public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
// logger.info("remove binding subscription value object if any...");
// ClientIdentity clientIdentity = (ClientIdentity) ctx.getAttachment();
// // 如果唯一的订阅者都取消了订阅，直接关闭服务，针对内部版本模式下可以减少资源浪费
// if (clientIdentity != null) {
// stopCanalInstanceIfNecessary(clientIdentity);
// }
}
private void stopCanalInstanceIfNecessary(ClientIdentity clientIdentity) {
List<ClientIdentity> clientIdentitys = embededServer.listAllSubscribe(clientIdentity.getDestination());
if (clientIdentitys != null && clientIdentitys.size() == 1 && clientIdentitys.contains(clientIdentity)) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (runningMonitor.isStart()) {
runningMonitor.release();
}
}
}
private TimeUnit convertTimeUnit(int unit) {
switch (unit) {
case 0:
return TimeUnit.NANOSECONDS;
case 1:
return TimeUnit.MICROSECONDS;
case 2:
return TimeUnit.MILLISECONDS;
case 3:
return TimeUnit.SECONDS;
case 4:
return TimeUnit.MINUTES;
case 5:
return TimeUnit.HOURS;
case 6:
return TimeUnit.DAYS;
default:
return TimeUnit.MILLISECONDS;
}
}
public void setEmbededServer(CanalServerWithEmbeded embededServer) {
this.embededServer = embededServer;
}
}

197
server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java Normal file
View File

@@ -0,0 +1,197 @@
package com.alibaba.otter.canal.server;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.CanalHASwitchable;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
public abstract class BaseCanalServerWithEmbededTest {
protected static final String cluster1 = "127.0.0.1:2188";
protected static final String DESTINATION = "ljhtest1";
protected static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
protected static final String MYSQL_ADDRESS = "127.0.0.1";
protected static final String USERNAME = "retl";
protected static final String PASSWORD = "retl";
protected static final String FILTER = "retl\\..*,erosa.zk_complaint_bizdata";
private CanalServerWithEmbeded server;
private ClientIdentity clientIdentity = new ClientIdentity(DESTINATION, (short) 1); ;
@Before
public void setUp() {
server = new CanalServerWithEmbeded();
server.setCanalInstanceGenerator(new CanalInstanceGenerator() {
public CanalInstance generate(String destination) {
Canal canal = buildCanal();
return new CanalInstanceWithManager(canal, FILTER);
}
});
server.start();
server.start(DESTINATION);
}
@After
public void tearDown() {
server.stop();
}
@Test
public void testGetWithoutAck() {
int maxEmptyCount = 10;
int emptyCount = 0;
int totalCount = 0;
server.subscribe(clientIdentity);
while (emptyCount < maxEmptyCount) {
Message message = server.getWithoutAck(clientIdentity, 11);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}
System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
server.ack(clientIdentity, message.getId());
}
}
System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
server.unsubscribe(clientIdentity);
}
@Test
public void testGet() {
int maxEmptyCount = 10;
int emptyCount = 0;
int totalCount = 0;
server.subscribe(clientIdentity);
while (emptyCount < maxEmptyCount) {
Message message = server.get(clientIdentity, 11);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}
System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
}
}
System.out.println("!!!!!! testGet totalCount : " + totalCount);
server.unsubscribe(clientIdentity);
}
// @Test
public void testRollback() {
int maxEmptyCount = 10;
int emptyCount = 0;
int totalCount = 0;
server.subscribe(clientIdentity);
while (emptyCount < maxEmptyCount) {
Message message = server.getWithoutAck(clientIdentity, 11);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}
System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
}
}
System.out.println("!!!!!! testRollback totalCount : " + totalCount);
server.rollback(clientIdentity);// 直接rollback掉，再取一次
emptyCount = 0;
totalCount = 0;
while (emptyCount < maxEmptyCount) {
Message message = server.getWithoutAck(clientIdentity, 11);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}
System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
}
}
System.out.println("!!!!!! testRollback after rollback , totalCount : " + totalCount);
server.unsubscribe(clientIdentity);
}
// @Test
public void testSwitch() {
int maxEmptyCount = 10;
int emptyCount = 0;
int totalCount = 0;
int thresold = 50;
int batchSize = 11;
server.subscribe(clientIdentity);
while (emptyCount < maxEmptyCount) {
Message message = server.get(clientIdentity, batchSize);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}
System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
if ((totalCount + 1) % 100 >= thresold && (totalCount + 1) % 100 <= thresold + batchSize) {
CanalEventParser eventParser = server.getCanalInstances().get(DESTINATION).getEventParser();
if (eventParser instanceof CanalHASwitchable) {
((CanalHASwitchable) eventParser).doSwitch();// 执行切换
try {
Thread.sleep(5 * 1000); // 等待parser启动
} catch (InterruptedException e) {
Assert.fail();
}
}
}
}
}
System.out.println("!!!!!! testGet totalCount : " + totalCount);
server.unsubscribe(clientIdentity);
}
abstract protected Canal buildCanal();
}

56
server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandaloneTest.java Normal file
View File

@@ -0,0 +1,56 @@
package com.alibaba.otter.canal.server;
import java.net.InetSocketAddress;
import java.util.Arrays;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
public class CanalServerWithEmbeded_StandaloneTest extends BaseCanalServerWithEmbededTest {
protected Canal buildCanal() {
Canal canal = new Canal();
canal.setId(1L);
canal.setName(DESTINATION);
canal.setDesc("test");
CanalParameter parameter = new CanalParameter();
parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
parameter.setMetaMode(MetaMode.MEMORY);
parameter.setHaMode(HAMode.HEARTBEAT);
parameter.setIndexMode(IndexMode.MEMORY);
parameter.setStorageMode(StorageMode.MEMORY);
parameter.setMemoryStorageBufferSize(32 * 1024);
parameter.setSourcingType(SourcingType.MYSQL);
parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
new InetSocketAddress(MYSQL_ADDRESS, 3306)));
parameter.setDbUsername(USERNAME);
parameter.setDbPassword(PASSWORD);
parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
"{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));
parameter.setSlaveId(1234L);
parameter.setDefaultConnectionTimeoutInSeconds(30);
parameter.setConnectionCharset("UTF-8");
parameter.setConnectionCharsetNumber((byte) 33);
parameter.setReceiveBufferSize(8 * 1024);
parameter.setSendBufferSize(8 * 1024);
parameter.setDetectingEnable(false);
parameter.setDetectingIntervalInSeconds(10);
parameter.setDetectingRetryTimes(3);
parameter.setDetectingSQL(DETECTING_SQL);
canal.setCanalParameter(parameter);
return canal;
}
}

68
server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandbyTest.java Normal file
View File

@@ -0,0 +1,68 @@
package com.alibaba.otter.canal.server;
import java.net.InetSocketAddress;
import java.util.Arrays;
import org.I0Itec.zkclient.ZkClient;
import org.junit.Before;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
public class CanalServerWithEmbeded_StandbyTest extends BaseCanalServerWithEmbededTest {
private ZkClient zkClient = new ZkClient(cluster1);
@Before
public void setUp() {
zkClient.deleteRecursive(ZookeeperPathUtils.CANAL_ROOT_NODE);
super.setUp();
}
protected Canal buildCanal() {
Canal canal = new Canal();
canal.setId(1L);
canal.setName(DESTINATION);
canal.setDesc("test");
CanalParameter parameter = new CanalParameter();
parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
parameter.setMetaMode(MetaMode.MIXED); // 冷备，可选择混合模式
parameter.setHaMode(HAMode.HEARTBEAT);
parameter.setIndexMode(IndexMode.META);// 内存版store，需要选择meta做为index
parameter.setStorageMode(StorageMode.MEMORY);
parameter.setMemoryStorageBufferSize(32 * 1024);
parameter.setSourcingType(SourcingType.MYSQL);
parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
new InetSocketAddress(MYSQL_ADDRESS, 3306)));
parameter.setDbUsername(USERNAME);
parameter.setDbPassword(PASSWORD);
parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
"{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));
parameter.setSlaveId(1234L);
parameter.setDefaultConnectionTimeoutInSeconds(30);
parameter.setConnectionCharset("UTF-8");
parameter.setConnectionCharsetNumber((byte) 33);
parameter.setReceiveBufferSize(8 * 1024);
parameter.setSendBufferSize(8 * 1024);
parameter.setDetectingEnable(false);
parameter.setDetectingIntervalInSeconds(10);
parameter.setDetectingRetryTimes(3);
parameter.setDetectingSQL(DETECTING_SQL);
canal.setCanalParameter(parameter);
return canal;
}
}

263
server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java Normal file
View File

@@ -0,0 +1,263 @@
package com.alibaba.otter.canal.server;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import java.util.Arrays;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
import com.alibaba.otter.canal.protocol.CanalPacket.Get;
import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
public class CanalServerWithNettyTest {
protected static final String cluster1 = "127.0.0.1:2188";
protected static final String DESTINATION = "ljhtest1";
protected static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
protected static final String MYSQL_ADDRESS = "127.0.0.1";
protected static final String USERNAME = "retl";
protected static final String PASSWORD = "retl";
protected static final String FILTER = "retl\\..*,erosa.canaltable1s,erosa.canaltable1t";
private final ByteBuffer header = ByteBuffer.allocate(4);
private CanalServerWithNetty nettyServer;
@Before
public void setUp() {
CanalServerWithEmbeded embededServer = new CanalServerWithEmbeded();
embededServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {
public CanalInstance generate(String destination) {
Canal canal = buildCanal();
return new CanalInstanceWithManager(canal, FILTER);
}
});
nettyServer = new CanalServerWithNetty(embededServer);
nettyServer.setPort(1088);
nettyServer.start();
}
@Test
public void testAuth() {
try {
SocketChannel channel = SocketChannel.open();
channel.connect(new InetSocketAddress("127.0.0.1", 1088));
Packet p = Packet.parseFrom(readNextPacket(channel));
if (p.getVersion() != 1) {
throw new Exception("unsupported version at this client.");
}
if (p.getType() != PacketType.HANDSHAKE) {
throw new Exception("expect handshake but found other type.");
}
//
Handshake handshake = Handshake.parseFrom(p.getBody());
System.out.println(handshake.getSupportedCompressionsList());
//
ClientAuth ca = ClientAuth.newBuilder()
.setUsername("")
.setNetReadTimeout(10000)
.setNetWriteTimeout(10000)
.build();
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTAUTHENTICATION)
.setBody(ca.toByteString())
.build()
.toByteArray());
//
p = Packet.parseFrom(readNextPacket(channel));
if (p.getType() != PacketType.ACK) {
throw new Exception("unexpected packet type when ack is expected");
}
Ack ack = Ack.parseFrom(p.getBody());
if (ack.getErrorCode() > 0) {
throw new Exception("something goes wrong when doing authentication: " + ack.getErrorMessage());
}
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.SUBSCRIPTION)
.setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
//
p = Packet.parseFrom(readNextPacket(channel));
ack = Ack.parseFrom(p.getBody());
if (ack.getErrorCode() > 0) {
throw new Exception("failed to subscribe with reason: " + ack.getErrorMessage());
}
for (int i = 0; i < 10; i++) {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.GET)
.setBody(Get.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
.setFetchSize(10)
.build()
.toByteString())
.build()
.toByteArray());
p = Packet.parseFrom(readNextPacket(channel));
long batchId = -1L;
switch (p.getType()) {
case MESSAGES: {
Messages messages = Messages.parseFrom(p.getBody());
batchId = messages.getBatchId();
break;
}
case ACK: {
ack = Ack.parseFrom(p.getBody());
if (ack.getErrorCode() > 0) {
throw new Exception("failed to subscribe with reason: " + ack.getErrorMessage());
}
break;
}
default: {
throw new Exception("unexpected packet type: " + p.getType());
}
}
System.out.println("!!!!!!!!!!!!!!!!! " + batchId);
Thread.sleep(1000L);
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTACK)
.setBody(ClientAck.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
.setBatchId(batchId)
.build()
.toByteString())
.build()
.toByteArray());
}
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTROLLBACK)
.setBody(ClientRollback.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
.build()
.toByteString())
.build()
.toByteArray());
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.UNSUBSCRIPTION)
.setBody(Unsub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
} catch (Exception e) {
e.printStackTrace();
}
}
@After
public void tearDown() {
nettyServer.stop();
}
private byte[] readNextPacket(SocketChannel channel) throws IOException {
header.clear();
read(channel, header);
int bodyLen = header.getInt(0);
ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen);
read(channel, bodyBuf);
return bodyBuf.array();
}
private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
ByteBuffer header = ByteBuffer.allocate(4);
header.putInt(body.length);
header.flip();
int len = channel.write(header);
assert (len == header.capacity());
channel.write(ByteBuffer.wrap(body));
}
private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
while (buffer.hasRemaining()) {
int r = channel.read(buffer);
if (r == -1) {
throw new IOException("end of stream when reading header");
}
}
}
private Canal buildCanal() {
Canal canal = new Canal();
canal.setId(1L);
canal.setName(DESTINATION);
canal.setDesc("test");
CanalParameter parameter = new CanalParameter();
parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
parameter.setMetaMode(MetaMode.MEMORY);
parameter.setHaMode(HAMode.HEARTBEAT);
parameter.setIndexMode(IndexMode.MEMORY);
parameter.setStorageMode(StorageMode.MEMORY);
parameter.setMemoryStorageBufferSize(32 * 1024);
parameter.setSourcingType(SourcingType.MYSQL);
parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
new InetSocketAddress(MYSQL_ADDRESS, 3306)));
parameter.setDbUsername(USERNAME);
parameter.setDbPassword(PASSWORD);
parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
"{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));
parameter.setSlaveId(1234L);
parameter.setDefaultConnectionTimeoutInSeconds(30);
parameter.setConnectionCharset("UTF-8");
parameter.setConnectionCharsetNumber((byte) 33);
parameter.setReceiveBufferSize(8 * 1024);
parameter.setSendBufferSize(8 * 1024);
parameter.setDetectingEnable(false);
parameter.setDetectingIntervalInSeconds(10);
parameter.setDetectingRetryTimes(3);
parameter.setDetectingSQL(DETECTING_SQL);
canal.setCanalParameter(parameter);
return canal;
}
}