chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #2099 , support user/passwd acl and canal admin operator

Browse Source
This commit is contained in:
agapple
2019-08-26 18:16:33 +08:00
parent e376a129ed
commit 3d4dbc9a8c
54 changed files with 8518 additions and 813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
server/src/main/java/com/alibaba/otter/canal/admin/CanalAdmin.java Normal file
View File

@@ -0,0 +1,110 @@
package com.alibaba.otter.canal.admin;
/**
* Canal Admin动态管理接口
*
* @author agapple 2019年8月24日 下午9:45:49
* @since 1.1.4
*/
public interface CanalAdmin {
/**
* 校验账号密码
*/
boolean auth(String user, String passwd, byte[] seed);
/**
* 获取Canal Server状态, 1代表运行/0代表不运行
*
* @return 状态代码
*/
boolean check();
/**
* 启动Canal Server
*
* @return 是否成功
*/
boolean start();
/**
* 停止Canal Server
*
* @return 是否成功
*/
boolean stop();
/**
* 重启Canal Server
*
* @return 是否成功
*/
boolean restart();
/**
* 获取所有当前节点下运行中的实例
*
* @return 实例信息
*/
String getRunningInstances();
/**
* 通过实例名检查
*
* @param destination
* @return
*/
boolean checkInstance(String destination);
/**
* 通过实例名启动实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean startInstance(String destination);
/**
* 通过实例名关闭实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean stopInstance(String destination);
/**
* 通过实例名重启实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean restartInstance(String destination);
/**
* 获取Canal Server日志列表
*
* @return 日志信息
*/
String listCanalLog();
/**
* 获取Canal Server日志
*
* @return 日志信息
*/
String canalLog(int lines);
/**
* 获取Instance的机器日志列表
*
* @param destination
*/
String listInstanceLog(String destination);
/**
* 通过实例名获取实例日志
*
* @return 日志信息
*/
String instanceLog(String destination, String fileName, int lines);
}

119
server/src/main/java/com/alibaba/otter/canal/admin/handler/ClientAuthenticationHandler.java Normal file
View File

@@ -0,0 +1,119 @@
package com.alibaba.otter.canal.admin.handler;
import java.util.concurrent.TimeUnit;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateEvent;
import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.admin.netty.AdminNettyUtils;
import com.alibaba.otter.canal.protocol.AdminPacket.ClientAuth;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.alibaba.otter.canal.server.netty.NettyUtils;
/**
* 客户端身份认证处理
*
* @author agapple 2019年8月24日 下午10:58:53
* @since 1.1.4
*/
public class ClientAuthenticationHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(ClientAuthenticationHandler.class);
private final int SUPPORTED_VERSION = 3;
private final int defaultSubscriptorDisconnectIdleTimeout = 60 * 60 * 1000;
private CanalAdmin canalAdmin;
private byte[] seed;
public ClientAuthenticationHandler(){
}
public ClientAuthenticationHandler(CanalAdmin canalAdmin){
this.canalAdmin = canalAdmin;
}
public void messageReceived(final ChannelHandlerContext ctx, MessageEvent e) throws Exception {
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
final Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
switch (packet.getVersion()) {
case SUPPORTED_VERSION:
default:
final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
if (seed == null) {
byte[] errorBytes = AdminNettyUtils.errorPacket(300,
MessageFormatter.format("auth failed for seed is null", clientAuth.getUsername()).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
}
if (!canalAdmin.auth(clientAuth.getUsername(), clientAuth.getPassword().toStringUtf8(), seed)) {
byte[] errorBytes = AdminNettyUtils.errorPacket(300,
MessageFormatter.format("auth failed for user:{}", clientAuth.getUsername()).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
}
byte[] ackBytes = AdminNettyUtils.ackPacket();
AdminNettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
logger.info("remove unused channel handlers after authentication is done successfully.");
ctx.getPipeline().remove(HandshakeInitializationHandler.class.getName());
ctx.getPipeline().remove(ClientAuthenticationHandler.class.getName());
int readTimeout = defaultSubscriptorDisconnectIdleTimeout;
int writeTimeout = defaultSubscriptorDisconnectIdleTimeout;
if (clientAuth.getNetReadTimeout() > 0) {
readTimeout = clientAuth.getNetReadTimeout();
}
if (clientAuth.getNetWriteTimeout() > 0) {
writeTimeout = clientAuth.getNetWriteTimeout();
}
// fix bug: soTimeout parameter's unit from connector is
// millseconds.
IdleStateHandler idleStateHandler = new IdleStateHandler(NettyUtils.hashedWheelTimer,
readTimeout,
writeTimeout,
0,
TimeUnit.MILLISECONDS);
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateHandler.class.getName(),
idleStateHandler);
IdleStateAwareChannelHandler idleStateAwareChannelHandler = new IdleStateAwareChannelHandler() {
public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) throws Exception {
logger.warn("channel:{} idle timeout exceeds, close channel to save server resources...",
ctx.getChannel());
ctx.getChannel().close();
}
};
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateAwareChannelHandler.class.getName(),
idleStateAwareChannelHandler);
}
});
break;
}
}
public void setCanalAdmin(CanalAdmin canalAdmin) {
this.canalAdmin = canalAdmin;
}
public void setSeed(byte[] seed) {
this.seed = seed;
}
}

62
server/src/main/java/com/alibaba/otter/canal/admin/handler/HandshakeInitializationHandler.java Normal file
View File

@@ -0,0 +1,62 @@
package com.alibaba.otter.canal.admin.handler;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.channel.group.ChannelGroup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.netty.AdminNettyUtils;
import com.alibaba.otter.canal.protocol.AdminPacket;
import com.alibaba.otter.canal.protocol.AdminPacket.Handshake;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.google.protobuf.ByteString;
/**
* handshake交互
*
* @author agapple 2019年8月24日 下午10:58:34
* @since 1.1.4
*/
public class HandshakeInitializationHandler extends SimpleChannelHandler {
// support to maintain socket channel.
private ChannelGroup childGroups;
public HandshakeInitializationHandler(ChannelGroup childGroups){
this.childGroups = childGroups;
}
private static final Logger logger = LoggerFactory.getLogger(HandshakeInitializationHandler.class);
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
// add new socket channel in channel container, used to manage sockets.
if (childGroups != null) {
childGroups.add(ctx.getChannel());
}
final byte[] seed = org.apache.commons.lang3.RandomUtils.nextBytes(8);
byte[] body = Packet.newBuilder()
.setType(AdminPacket.PacketType.HANDSHAKE)
.setVersion(AdminNettyUtils.VERSION)
.setBody(Handshake.newBuilder().setSeeds(ByteString.copyFrom(seed)).build().toByteString())
.build()
.toByteArray();
AdminNettyUtils.write(ctx.getChannel(), body, new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
logger.info("remove unused channel handlers after authentication is done successfully.");
ctx.getPipeline().get(HandshakeInitializationHandler.class.getName());
ClientAuthenticationHandler handler = (ClientAuthenticationHandler) ctx.getPipeline()
.get(ClientAuthenticationHandler.class.getName());
handler.setSeed(seed);
}
});
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}
}

154
server/src/main/java/com/alibaba/otter/canal/admin/handler/SessionHandler.java Normal file
View File

@@ -0,0 +1,154 @@
package com.alibaba.otter.canal.admin.handler;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.admin.netty.AdminNettyUtils;
import com.alibaba.otter.canal.protocol.AdminPacket.InstanceAdmin;
import com.alibaba.otter.canal.protocol.AdminPacket.LogAdmin;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.alibaba.otter.canal.protocol.AdminPacket.ServerAdmin;
public class SessionHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class);
private CanalAdmin canalAdmin;
public SessionHandler(){
}
public SessionHandler(CanalAdmin canalAdmin){
this.canalAdmin = canalAdmin;
}
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
logger.info("message receives in session handler...");
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
try {
String action = null;
String message = null;
String destination = null;
switch (packet.getType()) {
case SERVER:
ServerAdmin serverAdmin = ServerAdmin.parseFrom(packet.getBody());
action = serverAdmin.getAction();
switch (action) {
case "check":
message = canalAdmin.check() ? "1" : "0";
break;
case "start":
message = canalAdmin.start() ? "1" : "0";
break;
case "stop":
message = canalAdmin.stop() ? "1" : "0";
break;
case "restart":
message = canalAdmin.restart() ? "1" : "0";
break;
case "list":
message = canalAdmin.getRunningInstances();
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(301,
MessageFormatter.format("ServerAdmin action={} is unknown", action).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
AdminNettyUtils.write(ctx.getChannel(), AdminNettyUtils.ackPacket(message));
break;
case INSTANCE:
InstanceAdmin instanceAdmin = InstanceAdmin.parseFrom(packet.getBody());
destination = instanceAdmin.getDestination();
action = instanceAdmin.getAction();
switch (action) {
case "check":
message = canalAdmin.checkInstance(destination) ? "1" : "0";
break;
case "start":
message = canalAdmin.startInstance(destination) ? "1" : "0";
break;
case "stop":
message = canalAdmin.stopInstance(destination) ? "1" : "0";
break;
case "restart":
message = canalAdmin.restartInstance(destination) ? "1" : "0";
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(301,
MessageFormatter.format("InstanceAdmin action={} is unknown", action).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
AdminNettyUtils.write(ctx.getChannel(), AdminNettyUtils.ackPacket(message));
break;
case LOG:
LogAdmin logAdmin = LogAdmin.parseFrom(packet.getBody());
action = logAdmin.getAction();
destination = logAdmin.getDestination();
String type = logAdmin.getType();
String file = logAdmin.getFile();
int count = logAdmin.getCount();
switch (type) {
case "server":
if ("list".equalsIgnoreCase(action)) {
message = canalAdmin.listCanalLog();
} else {
message = canalAdmin.canalLog(count);
}
break;
case "instance":
if ("list".equalsIgnoreCase(action)) {
message = canalAdmin.listInstanceLog(destination);
} else {
message = canalAdmin.instanceLog(destination, file, count);
}
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(301,
MessageFormatter.format("LogAdmin type={} is unknown", type).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
AdminNettyUtils.write(ctx.getChannel(), AdminNettyUtils.ackPacket(message));
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(300,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
} catch (Throwable exception) {
byte[] errorBytes = AdminNettyUtils.errorPacket(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(exception)).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
}
}
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
logger.error("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(e.getCause()));
ctx.getChannel().close();
}
public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
}
public void setCanalAdmin(CanalAdmin canalAdmin) {
this.canalAdmin = canalAdmin;
}
}

63
server/src/main/java/com/alibaba/otter/canal/admin/netty/AdminNettyUtils.java Normal file
View File

@@ -0,0 +1,63 @@
package com.alibaba.otter.canal.admin.netty;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.buffer.CompositeChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.Channels;
import com.alibaba.otter.canal.protocol.AdminPacket;
import com.alibaba.otter.canal.protocol.AdminPacket.Ack;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
public class AdminNettyUtils {
public static int HEADER_LENGTH = 4;
public static int VERSION = 1;
public static void write(Channel channel, ByteBuffer body) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.limit()).array();
List<ChannelBuffer> components = new ArrayList<ChannelBuffer>(2);
components.add(ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, header));
components.add(ChannelBuffers.wrappedBuffer(body));
Channels.write(channel, new CompositeChannelBuffer(ByteOrder.BIG_ENDIAN, components));
}
public static void write(Channel channel, byte[] body) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body));
}
public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body)).addListener(channelFutureListner);
}
public static byte[] ackPacket() {
return ackPacket(null);
}
public static byte[] ackPacket(String message) {
return Packet.newBuilder()
.setType(AdminPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setCode(0).setMessage(message == null ? "" : message).build().toByteString())
.build()
.toByteArray();
}
public static byte[] errorPacket(int errorCode, String errorMessage) {
return Packet.newBuilder()
.setType(AdminPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setCode(errorCode).setMessage(errorMessage).build().toByteString())
.build()
.toByteArray();
}
}

127
server/src/main/java/com/alibaba/otter/canal/admin/netty/CanalAdminWithNetty.java Normal file
View File

@@ -0,0 +1,127 @@
package com.alibaba.otter.canal.admin.netty;
import java.net.InetSocketAddress;
import java.util.concurrent.Executors;
import org.apache.commons.lang.StringUtils;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.group.ChannelGroup;
import org.jboss.netty.channel.group.DefaultChannelGroup;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.admin.handler.ClientAuthenticationHandler;
import com.alibaba.otter.canal.admin.handler.HandshakeInitializationHandler;
import com.alibaba.otter.canal.admin.handler.SessionHandler;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.server.netty.handler.FixedHeaderFrameDecoder;
/**
* 基于netty网络服务的server实现
*
* @author jianghang 2012-7-12 下午01:34:49
* @version 1.0.0
*/
public class CanalAdminWithNetty extends AbstractCanalLifeCycle {
private String ip;
private int port;
private Channel serverChannel = null;
private ServerBootstrap bootstrap = null;
private ChannelGroup childGroups = null; // socket channel
// container, used to
// close sockets
// explicitly.
private CanalAdmin canalAdmin;
private static class SingletonHolder {
private static final CanalAdminWithNetty CANAL_ADMIN_WITH_NETTY = new CanalAdminWithNetty();
}
private CanalAdminWithNetty(){
this.childGroups = new DefaultChannelGroup();
}
public static CanalAdminWithNetty instance() {
return SingletonHolder.CANAL_ADMIN_WITH_NETTY;
}
public void start() {
super.start();
this.bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),
Executors.newCachedThreadPool()));
/*
* enable keep-alive mechanism, handle abnormal network connection
* scenarios on OS level. the threshold parameters are depended on OS.
* e.g. On Linux: net.ipv4.tcp_keepalive_time = 300
* net.ipv4.tcp_keepalive_probes = 2 net.ipv4.tcp_keepalive_intvl = 30
*/
bootstrap.setOption("child.keepAlive", true);
/*
* optional parameter.
*/
bootstrap.setOption("child.tcpNoDelay", true);
// 构造对应的pipeline
bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipelines = Channels.pipeline();
pipelines.addLast(FixedHeaderFrameDecoder.class.getName(), new FixedHeaderFrameDecoder());
// support to maintain child socket channel.
pipelines.addLast(HandshakeInitializationHandler.class.getName(),
new HandshakeInitializationHandler(childGroups));
pipelines.addLast(ClientAuthenticationHandler.class.getName(),
new ClientAuthenticationHandler(canalAdmin));
SessionHandler sessionHandler = new SessionHandler(canalAdmin);
pipelines.addLast(SessionHandler.class.getName(), sessionHandler);
return pipelines;
}
});
// 启动
if (StringUtils.isNotEmpty(ip)) {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.ip, this.port));
} else {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.port));
}
}
public void stop() {
super.stop();
if (this.serverChannel != null) {
this.serverChannel.close().awaitUninterruptibly(1000);
}
// close sockets explicitly to reduce socket channel hung in complicated
// network environment.
if (this.childGroups != null) {
this.childGroups.close().awaitUninterruptibly(5000);
}
if (this.bootstrap != null) {
this.bootstrap.releaseExternalResources();
}
}
public void setIp(String ip) {
this.ip = ip;
}
public void setPort(int port) {
this.port = port;
}
public void setCanalAdmin(CanalAdmin canalAdmin) {
this.canalAdmin = canalAdmin;
}
}

2
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java
View File

@@ -9,6 +9,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
@@ -66,6 +67,7 @@ public class CanalMessageSerializer {
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setVersion(NettyUtils.VERSION);
packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray();
}

35
server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
View File

@@ -1,12 +1,15 @@
package com.alibaba.otter.canal.server.embedded;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -18,6 +21,7 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.SecurityUtil;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
@@ -52,6 +56,8 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
private CanalInstanceGenerator canalInstanceGenerator;
private int metricsPort;
private CanalMetricsService metrics = NopCanalMetricsService.NOP;
private String user;
private String passwd;
private static class SingletonHolder {
@@ -106,6 +112,27 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
metrics.terminate();
}
public boolean auth(String user, String passwd, byte[] seed) {
// 如果user/passwd密码为空,则任何用户账户都能登录
if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
if (StringUtils.isEmpty(this.passwd)) {
return true;
} else if (StringUtils.isEmpty(passwd)) {
// 如果server密码有配置,客户端密码为空,则拒绝
return false;
}
try {
byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd);
return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed);
} catch (NoSuchAlgorithmException e) {
return false;
}
}
return false;
}
public void start(final String destination) {
final CanalInstance canalInstance = canalInstances.get(destination);
if (!canalInstance.isStart()) {
@@ -572,4 +599,12 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
this.metricsPort = metricsPort;
}
public void setUser(String user) {
this.user = user;
}
public void setPasswd(String passwd) {
this.passwd = passwd;
}
}

21
server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
View File

@@ -25,6 +25,7 @@ public class NettyUtils {
private static final Logger logger = LoggerFactory.getLogger(NettyUtils.class);
public static int HEADER_LENGTH = 4;
public static Timer hashedWheelTimer = new HashedWheelTimer();
public static int VERSION = 1;
public static void write(Channel channel, ByteBuffer body, ChannelFutureListener channelFutureListner) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.limit()).array();
@@ -53,6 +54,7 @@ public class NettyUtils {
write(channel,
Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray(),
@@ -69,6 +71,7 @@ public class NettyUtils {
write(channel,
Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray(),
@@ -77,17 +80,19 @@ public class NettyUtils {
public static byte[] ackPacket() {
return Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray();
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray();
}
public static byte[] errorPacket(int errorCode, String errorMessage) {
return Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray();
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray();
}
}

19
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java
View File

@@ -15,6 +15,7 @@ import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.slf4j.helpers.MessageFormatter;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
@@ -36,6 +37,7 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
private final int SUPPORTED_VERSION = 3;
private final int defaultSubscriptorDisconnectIdleTimeout = 60 * 60 * 1000;
private CanalServerWithEmbedded embeddedServer;
private byte[] seed;
public ClientAuthenticationHandler(){
@@ -52,6 +54,18 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
case SUPPORTED_VERSION:
default:
final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
if (seed == null) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("auth failed for seed is null", clientAuth.getUsername()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, null);
}
if (!embeddedServer.auth(clientAuth.getUsername(), clientAuth.getPassword().toStringUtf8(), seed)) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("auth failed for user:{}", clientAuth.getUsername()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, null);
}
// 如果存在订阅信息
if (StringUtils.isNotEmpty(clientAuth.getDestination())
&& StringUtils.isNotEmpty(clientAuth.getClientId())) {
@@ -61,7 +75,6 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
try {
MDC.put("destination", clientIdentity.getDestination());
embeddedServer.subscribe(clientIdentity);
ctx.setAttachment(clientIdentity);// 设置状态数据
// 尝试启动，如果已经启动，忽略
if (!embeddedServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
@@ -123,4 +136,8 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
this.embeddedServer = embeddedServer;
}
public void setSeed(byte[] seed) {
this.seed = seed;
}
}

19
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java
View File

@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.server.netty.handler;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
@@ -11,6 +13,7 @@ import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString;
/**
* handshake交互
@@ -35,12 +38,24 @@ public class HandshakeInitializationHandler extends SimpleChannelHandler {
childGroups.add(ctx.getChannel());
}
final byte[] seed = org.apache.commons.lang3.RandomUtils.nextBytes(8);
byte[] body = Packet.newBuilder()
.setType(CanalPacket.PacketType.HANDSHAKE)
.setBody(Handshake.newBuilder().build().toByteString())
.setVersion(NettyUtils.VERSION)
.setBody(Handshake.newBuilder().setSeeds(ByteString.copyFrom(seed)).build().toByteString())
.build()
.toByteArray();
NettyUtils.write(ctx.getChannel(), body, null);
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
ctx.getPipeline().get(HandshakeInitializationHandler.class.getName());
ClientAuthenticationHandler handler = (ClientAuthenticationHandler) ctx.getPipeline()
.get(ClientAuthenticationHandler.class.getName());
handler.setSeed(seed);
}
});
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}
}

141
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
View File

@@ -79,11 +79,21 @@ public class SessionHandler extends SimpleChannelHandler {
// ctx.setAttachment(clientIdentity);// 设置状态数据
byte[] ackBytes = NettyUtils.ackPacket();
NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(sub.getDestination(),
sub, packet.getType(), ackBytes.length, System.nanoTime() - start));
sub,
packet.getType(),
ackBytes.length,
System.nanoTime() - start));
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes ,new ChannelFutureAggregator(sub.getDestination(),
sub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(sub.getDestination(),
sub,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case UNSUBSCRIPTION:
@@ -96,12 +106,24 @@ public class SessionHandler extends SimpleChannelHandler {
embeddedServer.unsubscribe(clientIdentity);
stopCanalInstanceIfNecessary(clientIdentity);// 尝试关闭
byte[] ackBytes = NettyUtils.ackPacket();
NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(unsub.getDestination(),
unsub, packet.getType(), ackBytes.length, System.nanoTime() - start));
NettyUtils.write(ctx.getChannel(),
ackBytes,
new ChannelFutureAggregator(unsub.getDestination(),
unsub,
packet.getType(),
ackBytes.length,
System.nanoTime() - start));
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(unsub.getDestination(),
unsub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(unsub.getDestination(),
unsub,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case GET:
@@ -177,15 +199,19 @@ public class SessionHandler extends SimpleChannelHandler {
}
output.checkNoSpaceLeft();
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(),
get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));
get,
packet.getType(),
body.length,
System.nanoTime() - start,
message.getId() == -1));
// output.flush();
// byteBuffer.flip();
// NettyUtils.write(ctx.getChannel(), byteBuffer,
// null);
} else {
Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setType(PacketType.MESSAGES).setVersion(NettyUtils.VERSION);
Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
messageBuilder.setBatchId(message.getId());
@@ -198,14 +224,27 @@ public class SessionHandler extends SimpleChannelHandler {
}
}
}
byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString()).build().toByteArray();
byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString())
.build()
.toByteArray();
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(),
get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));// 输出数据
get,
packet.getType(),
body.length,
System.nanoTime() - start,
message.getId() == -1));// 输出数据
}
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(get.getDestination(),
get, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(get.getDestination(),
get,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case CLIENTACK:
@@ -213,20 +252,38 @@ public class SessionHandler extends SimpleChannelHandler {
MDC.put("destination", ack.getDestination());
if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
if (ack.getBatchId() == 0L) {
byte[] errorBytes = NettyUtils.errorPacket(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(),
ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 402));
byte[] errorBytes = NettyUtils.errorPacket(402,
MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 402));
} else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据，直接忽略之
// donothing
} else {
clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
embeddedServer.ack(clientIdentity, ack.getBatchId());
new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
0,
System.nanoTime() - start).operationComplete(null);
}
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(),
ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case CLIENTROLLBACK:
@@ -241,25 +298,41 @@ public class SessionHandler extends SimpleChannelHandler {
} else {
embeddedServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次
}
new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
new ChannelFutureAggregator(rollback.getDestination(),
rollback,
packet.getType(),
0,
System.nanoTime() - start).operationComplete(null);
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(rollback.getDestination(),
rollback, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", rollback.toString())
.getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(rollback.getDestination(),
rollback,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
default:
byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(),
null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel()
.getRemoteAddress()
.toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
break;
}
} catch (Throwable exception) {
byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}",
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(exception)).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(),
null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel()
.getRemoteAddress()
.toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
} finally {
MDC.remove("destination");
}

20
server/src/test/java/com/alibaba/otter/canal/server/CanalServerTest.java
View File

@@ -34,6 +34,8 @@ import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
import com.alibaba.otter.canal.server.netty.NettyUtils;
@Ignore
public class CanalServerTest {
@@ -68,7 +70,6 @@ public class CanalServerTest {
@Test
public void testAuth() {
try {
SocketChannel channel = SocketChannel.open();
channel.connect(new InetSocketAddress("127.0.0.1", 1088));
@@ -93,6 +94,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTAUTHENTICATION)
.setVersion(NettyUtils.VERSION)
.setBody(ca.toByteString())
.build()
.toByteArray());
@@ -107,12 +109,12 @@ public class CanalServerTest {
throw new Exception("something goes wrong when doing authentication: " + ack.getErrorMessage());
}
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.SUBSCRIPTION)
.setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
writeWithHeader(channel, Packet.newBuilder()
.setType(PacketType.SUBSCRIPTION)
.setVersion(NettyUtils.VERSION)
.setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
//
p = Packet.parseFrom(readNextPacket(channel));
ack = Ack.parseFrom(p.getBody());
@@ -124,6 +126,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.GET)
.setVersion(NettyUtils.VERSION)
.setBody(Get.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
@@ -158,6 +161,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTACK)
.setVersion(NettyUtils.VERSION)
.setBody(ClientAck.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
@@ -171,6 +175,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTROLLBACK)
.setVersion(NettyUtils.VERSION)
.setBody(ClientRollback.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
@@ -182,6 +187,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.UNSUBSCRIPTION)
.setVersion(NettyUtils.VERSION)
.setBody(Unsub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());

22
server/src/test/java/com/alibaba/otter/canal/server/SecurityUtilTest.java Normal file
View File

@@ -0,0 +1,22 @@
package com.alibaba.otter.canal.server;
import java.security.NoSuchAlgorithmException;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.protocol.SecurityUtil;
public class SecurityUtilTest {
@Test
public void testSimple() throws NoSuchAlgorithmException {
byte[] seed = { 1, 2, 3, 4, 5, 6, 7, 8 };
// String str = "e3619321c1a937c46a0d8bd1dac39f93b27d4458"; // canal
// passwd
String str = SecurityUtil.scrambleGenPass("canal".getBytes());
byte[] client = SecurityUtil.scramble411("canal".getBytes(), seed);
boolean check = SecurityUtil.scrambleServerAuth(client, SecurityUtil.hexStr2Bytes(str), seed);
Assert.assertTrue(check);
}
}