From 5582a38930814d30b147285a4074fe93c72849a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 19 Jul 2018 13:35:38 +0800 Subject: [PATCH 01/37] =?UTF-8?q?fixed=20issue=20#726,=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96SessionHandler=E9=87=8C=E7=9B=B4=E6=8E=A5=E6=8B=BCbyte?= =?UTF-8?q?[],=E7=BB=95=E8=BF=87protobuf=E7=9A=84=E5=A4=9A=E6=AC=A1?= =?UTF-8?q?=E6=8B=B7=E8=B4=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/impl/SimpleCanalConnector.java | 3 +- .../parse/inbound/AbstractEventParser.java | 9 +- .../parse/inbound/MultiStageCoprocessor.java | 4 +- .../inbound/mysql/LocalBinLogConnection.java | 6 +- .../parse/inbound/mysql/MysqlConnection.java | 8 +- .../mysql/MysqlMultiStageCoprocessor.java | 16 ++-- .../otter/canal/server/netty/NettyUtils.java | 2 +- .../server/netty/handler/SessionHandler.java | 65 +++++++++---- .../otter/canal/server/ProtocolTest.java | 92 +++++++++++++++++++ 9 files changed, 169 insertions(+), 36 deletions(-) create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java index 508c01dc..0f29794d 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java @@ -319,7 +319,8 @@ public class SimpleCanalConnector implements CanalConnector { } private Message receiveMessages() throws IOException { - Packet p = Packet.parseFrom(readNextPacket()); + byte[] data = readNextPacket(); + Packet p = Packet.parseFrom(data); switch (p.getType()) { case MESSAGES: { if (!p.getCompression().equals(Compression.NONE)) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index 54bb14e7..c8cdb1bd 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -331,6 +331,11 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle stopHeartBeat(); // 先停止心跳 parseThread.interrupt(); // 尝试中断 eventSink.interrupt(); + + if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) { + multiStageCoprocessor.stop(); + } + try { parseThread.join();// 等待其结束 } catch (InterruptedException e) { @@ -343,10 +348,6 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle if (transactionBuffer.isStart()) { transactionBuffer.stop(); } - - if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) { - multiStageCoprocessor.stop(); - } } protected boolean consumeTheEventAndProfilingIfNecessary(List entrys) throws CanalSinkException, diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java index 1c5c495d..96e063e9 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java @@ -21,9 +21,9 @@ public interface MultiStageCoprocessor extends CanalLifeCycle { /** * 网络数据投递 */ - public void publish(LogBuffer buffer); + public boolean publish(LogBuffer buffer); - public void publish(LogBuffer buffer, String binlogFileName); + public boolean publish(LogBuffer buffer, String binlogFileName); public void reset(); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java index f97c4ccd..6c59465b 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java @@ -221,7 +221,11 @@ public class LocalBinLogConnection implements ErosaConnection { while (fetcher.fetch()) { LogBuffer buffer = fetcher.duplicate(); fetcher.consume(fetcher.limit()); - coprocessor.publish(buffer, binlogfilename); // set filename + // set filename + if (!coprocessor.publish(buffer, binlogfilename)) { + needContinue = false; + break; + } } if (needContinue) {// 读取下一个 diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 46f7e300..d01a210e 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -206,7 +206,9 @@ public class MysqlConnection implements ErosaConnection { while (fetcher.fetch()) { LogBuffer buffer = fetcher.duplicate(); fetcher.consume(fetcher.limit()); - coprocessor.publish(buffer); + if (!coprocessor.publish(buffer)) { + break; + } } } finally { fetcher.close(); @@ -230,7 +232,9 @@ public class MysqlConnection implements ErosaConnection { while (fetcher.fetch()) { LogBuffer buffer = fetcher.duplicate(); fetcher.consume(fetcher.limit()); - coprocessor.publish(buffer); + if (!coprocessor.publish(buffer)) { + break; + } } } finally { fetcher.close(); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 76c70723..65904b34 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -135,20 +135,22 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement } catch (Throwable e) { // ignore } - disruptorMsgBuffer = null; super.stop(); } /** * 网络数据投递 */ - public void publish(LogBuffer buffer) { - publish(buffer, null); + public boolean publish(LogBuffer buffer) { + return publish(buffer, null); } - public void publish(LogBuffer buffer, String binlogFileName) { - if (!isStart() && exception != null) { - throw exception; + public boolean publish(LogBuffer buffer, String binlogFileName) { + if (!isStart()) { + if (exception != null) { + throw exception; + } + return false; } boolean interupted = false; @@ -172,6 +174,8 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement if (exception != null) { throw exception; } + + return isStart(); } @Override diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java index 531c817f..22f45617 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java @@ -19,7 +19,7 @@ import com.alibaba.otter.canal.protocol.CanalPacket.Packet; public class NettyUtils { private static final Logger logger = LoggerFactory.getLogger(NettyUtils.class); - private static int HEADER_LENGTH = 4; + public static int HEADER_LENGTH = 4; public static Timer hashedWheelTimer = new HashedWheelTimer(); public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) { diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java index 8f7b4278..8b94d452 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java @@ -33,13 +33,10 @@ import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; import com.alibaba.otter.canal.server.netty.NettyUtils; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.WireFormat; -/** - * 处理具体的客户端请求 - * - * @author jianghang 2012-10-24 下午02:21:13 - * @version 1.0.0 - */ public class SessionHandler extends SimpleChannelHandler { private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class); @@ -130,27 +127,57 @@ public class SessionHandler extends SimpleChannelHandler { } // } - Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); - packetBuilder.setType(PacketType.MESSAGES); + if (message.getId() != -1 && message.isRaw()) { + List rowEntries = message.getRawEntries(); + // message size + int messageSize = 0; + messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId()); - Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); - messageBuilder.setBatchId(message.getId()); - if (message.getId() != -1) { - if (message.isRaw()) { - // for performance - if (!CollectionUtils.isEmpty(message.getRawEntries())) { + int dataSize = 0; + for (int i = 0; i < rowEntries.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i)); + } + messageSize += dataSize; + messageSize += 1 * rowEntries.size(); + // packet size + int size = 0; + size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, + PacketType.MESSAGES.getNumber()); + size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + + messageSize; + // TODO recyle bytes[] + byte[] body = new byte[size]; + CodedOutputStream output = CodedOutputStream.newInstance(body); + output.writeEnum(3, PacketType.MESSAGES.getNumber()); + + output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(messageSize); + // message + output.writeInt64(1, message.getId()); + for (int i = 0; i < rowEntries.size(); i++) { + output.writeBytes(2, rowEntries.get(i)); + } + output.checkNoSpaceLeft(); + NettyUtils.write(ctx.getChannel(), body, null); + } else { + Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); + packetBuilder.setType(PacketType.MESSAGES); + + Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); + messageBuilder.setBatchId(message.getId()); + if (message.getId() != -1) { + if (message.isRaw() && !CollectionUtils.isEmpty(message.getRawEntries())) { messageBuilder.addAllMessages(message.getRawEntries()); - } - } else { - if (!CollectionUtils.isEmpty(message.getEntries())) { + } else if (!CollectionUtils.isEmpty(message.getEntries())) { for (Entry entry : message.getEntries()) { messageBuilder.addMessages(entry.toByteString()); } } } + packetBuilder.setBody(messageBuilder.build().toByteString()); + NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据 } - packetBuilder.setBody(messageBuilder.build().toByteString()); - NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据 } else { NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(), diff --git a/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java b/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java new file mode 100644 index 00000000..cb078ddf --- /dev/null +++ b/server/src/test/java/com/alibaba/otter/canal/server/ProtocolTest.java @@ -0,0 +1,92 @@ +package com.alibaba.otter.canal.server; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.junit.Test; + +import com.alibaba.otter.canal.protocol.CanalEntry.Entry; +import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; +import com.alibaba.otter.canal.protocol.CanalEntry.Header; +import com.alibaba.otter.canal.protocol.CanalPacket.Compression; +import com.alibaba.otter.canal.protocol.CanalPacket.Messages; +import com.alibaba.otter.canal.protocol.CanalPacket.Packet; +import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.WireFormat; + +public class ProtocolTest { + + @Test + public void testSimple() throws IOException { + Header.Builder headerBuilder = Header.newBuilder(); + headerBuilder.setLogfileName("mysql-bin.000001"); + headerBuilder.setLogfileOffset(1024); + headerBuilder.setExecuteTime(1024); + Entry.Builder entryBuilder = Entry.newBuilder(); + entryBuilder.setHeader(headerBuilder.build()); + entryBuilder.setEntryType(EntryType.ROWDATA); + Entry entry = entryBuilder.build(); + Message message = new Message(3, true, Arrays.asList(entry.toByteString())); + + byte[] body = buildData(message); + Packet packet = Packet.parseFrom(body); + switch (packet.getType()) { + case MESSAGES: { + if (!packet.getCompression().equals(Compression.NONE)) { + throw new CanalClientException("compression is not supported in this connector"); + } + + Messages messages = Messages.parseFrom(packet.getBody()); + Message result = new Message(messages.getBatchId()); + for (ByteString byteString : messages.getMessagesList()) { + result.addEntry(Entry.parseFrom(byteString)); + } + + System.out.println(result); + break; + } + default: { + throw new CanalClientException("unexpected packet type: " + packet.getType()); + } + } + } + + private byte[] buildData(Message message) throws IOException { + List rowEntries = message.getRawEntries(); + // message size + int messageSize = 0; + messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId()); + + int dataSize = 0; + for (int i = 0; i < rowEntries.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i)); + } + messageSize += dataSize; + messageSize += 1 * rowEntries.size(); + // packet size + int size = 0; + size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber()); + size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize; + // TODO recyle bytes[] + byte[] body = new byte[size]; + CodedOutputStream output = CodedOutputStream.newInstance(body); + output.writeEnum(3, PacketType.MESSAGES.getNumber()); + + output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(messageSize); + // message + output.writeInt64(1, message.getId()); + for (int i = 0; i < rowEntries.size(); i++) { + output.writeBytes(2, rowEntries.get(i)); + } + output.checkNoSpaceLeft(); + + return body; + } +} From b19eaeb89a90b980e550e98b38e89efdd5ead222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 19 Jul 2018 17:55:53 +0800 Subject: [PATCH 02/37] =?UTF-8?q?fixed=20issue=20#726=20,=20=E5=8D=95?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E5=A4=8D=E7=94=A8ByteBuffer=E6=95=B0?= =?UTF-8?q?=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/server/netty/NettyUtils.java | 18 ++++++++++++++++ .../server/netty/handler/SessionHandler.java | 21 +++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java index 22f45617..20532400 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java @@ -2,8 +2,12 @@ package com.alibaba.otter.canal.server.netty; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; +import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.buffer.CompositeChannelBuffer; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFutureListener; import org.jboss.netty.channel.Channels; @@ -22,6 +26,20 @@ public class NettyUtils { public static int HEADER_LENGTH = 4; public static Timer hashedWheelTimer = new HashedWheelTimer(); + public static void write(Channel channel, ByteBuffer body, ChannelFutureListener channelFutureListner) { + byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.limit()).array(); + List components = new ArrayList(2); + components.add(ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, header)); + components.add(ChannelBuffers.wrappedBuffer(body)); + + if (channelFutureListner == null) { + Channels.write(channel, new CompositeChannelBuffer(ByteOrder.BIG_ENDIAN, components)); + } else { + Channels.write(channel, new CompositeChannelBuffer(ByteOrder.BIG_ENDIAN, components)) + .addListener(channelFutureListner); + } + } + public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) { byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array(); if (channelFutureListner == null) { diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java index 8b94d452..11bba043 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java @@ -73,7 +73,7 @@ public class SessionHandler extends SimpleChannelHandler { } embeddedServer.subscribe(clientIdentity); - ctx.setAttachment(clientIdentity);// 设置状态数据 + // ctx.setAttachment(clientIdentity);// 设置状态数据 NettyUtils.ack(ctx.getChannel(), null); } else { NettyUtils.error(401, @@ -146,7 +146,19 @@ public class SessionHandler extends SimpleChannelHandler { size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize; - // TODO recyle bytes[] + // recyle bytes + // ByteBuffer byteBuffer = (ByteBuffer) + // ctx.getAttachment(); + // if (byteBuffer != null && size <= + // byteBuffer.capacity()) { + // byteBuffer.clear(); + // } else { + // byteBuffer = + // ByteBuffer.allocate(size).order(ByteOrder.BIG_ENDIAN); + // ctx.setAttachment(byteBuffer); + // } + // CodedOutputStream output = + // CodedOutputStream.newInstance(byteBuffer); byte[] body = new byte[size]; CodedOutputStream output = CodedOutputStream.newInstance(body); output.writeEnum(3, PacketType.MESSAGES.getNumber()); @@ -160,6 +172,11 @@ public class SessionHandler extends SimpleChannelHandler { } output.checkNoSpaceLeft(); NettyUtils.write(ctx.getChannel(), body, null); + + // output.flush(); + // byteBuffer.flip(); + // NettyUtils.write(ctx.getChannel(), byteBuffer, + // null); } else { Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); packetBuilder.setType(PacketType.MESSAGES); From 617f9bbc624b11ebbb179d6ddecc73c37ba219a6 Mon Sep 17 00:00:00 2001 From: CAI1987 <41417665+CAI1987@users.noreply.github.com> Date: Thu, 19 Jul 2018 19:24:37 +0800 Subject: [PATCH 03/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Canal=E5=95=86=E4=B8=9A?= =?UTF-8?q?=E5=8C=96=E7=89=88=E6=9C=AC=E6=9B=B4=E6=96=B0=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 456bd2db..e90f7869 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@
  1. canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。
  2. canal消费端项目开源: Otter(分布式数据库同步系统)，地址：https://github.com/alibaba/otter
    3. +
  3. Canal已在阿里云推出商业化版本 https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E

背景

From b9156936ae98085204e8eea0005931598c143161 Mon Sep 17 00:00:00 2001 From: CAI1987 <41417665+CAI1987@users.noreply.github.com> Date: Thu, 19 Jul 2018 20:12:03 +0800 Subject: [PATCH 04/37] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=95=86=E4=B8=9A?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index e90f7869..736e238e 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,15 @@ See the wiki page for : wiki文
  • 阿里巴巴去Oracle数据迁移同步工具(目标支持MySQL/DRDS)：http://github.com/alibaba/yugong
    • +

    相关产品

    +
      +
    1. 阿里云分布式数据库DRDS阿里云分布式数据库DRDS
      2. +
    2. 阿里云数据传输服务DTS阿里云数据传输服务DTS
      3. +
    +
  • 阿里云数据库备份服务DBS
    • +
  • 阿里云数据管理服务DMS
    • + +

    问题反馈

    1. qq交流群： 161559791
      2. From b880d96a862c38965ce26f8766f28806aea2c1fa Mon Sep 17 00:00:00 2001 From: CAI1987 <41417665+CAI1987@users.noreply.github.com> Date: Thu, 19 Jul 2018 20:13:01 +0800 Subject: [PATCH 05/37] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=95=86=E4=B8=9A?= =?UTF-8?q?=E5=8C=96=E7=89=88=E6=9C=AC=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 736e238e..1ddbd9c7 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@
      1. canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。
      2. canal消费端项目开源: Otter(分布式数据库同步系统)，地址：https://github.com/alibaba/otter
        3. -
      3. Canal已在阿里云推出商业化版本 https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E
        4. +
      4. Canal已在阿里云推出商业化版本 数据传输服务DTS， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。DTS产品使用文档

      背景

      From b9d7f0b6d0bd9a2f6b6c7541e427bc0d6503ced5 Mon Sep 17 00:00:00 2001 From: CAI1987 <41417665+CAI1987@users.noreply.github.com> Date: Thu, 19 Jul 2018 20:14:19 +0800 Subject: [PATCH 06/37] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ddbd9c7..58b4cda0 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,8 @@ See the wiki page for : wiki文

      相关产品

        -
      1. 阿里云分布式数据库DRDS阿里云分布式数据库DRDS
        2. -
      2. 阿里云数据传输服务DTS阿里云数据传输服务DTS
        3. +
      3. 阿里云分布式数据库DRDS
        4. +
      4. 阿里云数据传输服务DTS
    2. 阿里云数据库备份服务DBS
    3. 阿里云数据管理服务DMS
      4. From 67f2e55d38c50686bb1ba7a68e7c6181617d5c46 Mon Sep 17 00:00:00 2001 From: CAI1987 <41417665+CAI1987@users.noreply.github.com> Date: Thu, 19 Jul 2018 20:14:49 +0800 Subject: [PATCH 07/37] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 58b4cda0..9e596f07 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ See the wiki page for : wiki文
      1. 阿里云分布式数据库DRDS
      2. 阿里云数据传输服务DTS
        3. -
    4. 阿里云数据库备份服务DBS
    5. 阿里云数据管理服务DMS
    From fe38bc16d2c95c4367e23fe06ceb628f20906b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 20 Jul 2018 14:32:13 +0800 Subject: [PATCH 08/37] =?UTF-8?q?fixed=20issue=20#726=20,=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=AE=8C=E6=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dbsync/binlog/DirectLogFetcherTest.java | 93 ++++++++----------- .../driver/mysql/socket/BioSocketChannel.java | 4 +- .../mysql/socket/BioSocketChannelPool.java | 2 + .../parse/inbound/mysql/MysqlDumpTest.java | 3 + 4 files changed, 48 insertions(+), 54 deletions(-) diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java index 23a10f01..78d5b586 100644 --- a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java @@ -8,15 +8,6 @@ import java.sql.Statement; import org.junit.Assert; import org.junit.Test; -import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; -import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; -import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; -import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent; -import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; -import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; -import com.taobao.tddl.dbsync.binlog.event.XidLogEvent; -import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; - public class DirectLogFetcherTest extends BaseLogFetcherTest { @Test @@ -24,56 +15,54 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest { DirectLogFetcher fecther = new DirectLogFetcher(); try { Class.forName("com.mysql.jdbc.Driver"); - Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello"); + Connection connection = DriverManager.getConnection("jdbc:mysql://100.81.154.142:3306", "root", "hello"); Statement statement = connection.createStatement(); statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'"); statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'"); - fecther.open(connection, "mysql-bin.000001", 4L, 2); + fecther.open(connection, "mysql-bin.000006", 120L, 2); - LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); + LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.UNKNOWN_EVENT); LogContext context = new LogContext(); while (fecther.fetch()) { - LogEvent event = null; - event = decoder.decode(fecther, context); - - if (event == null) { - continue; - // throw new RuntimeException("parse failed"); - } - - int eventType = event.getHeader().getType(); - switch (eventType) { - case LogEvent.ROTATE_EVENT: - binlogFileName = ((RotateLogEvent) event).getFilename(); - break; - case LogEvent.WRITE_ROWS_EVENT_V1: - case LogEvent.WRITE_ROWS_EVENT: - parseRowsEvent((WriteRowsLogEvent) event); - break; - case LogEvent.UPDATE_ROWS_EVENT_V1: - case LogEvent.UPDATE_ROWS_EVENT: - parseRowsEvent((UpdateRowsLogEvent) event); - break; - case LogEvent.DELETE_ROWS_EVENT_V1: - case LogEvent.DELETE_ROWS_EVENT: - parseRowsEvent((DeleteRowsLogEvent) event); - break; - case LogEvent.QUERY_EVENT: - parseQueryEvent((QueryLogEvent) event); - break; - case LogEvent.ROWS_QUERY_LOG_EVENT: - parseRowsQueryEvent((RowsQueryLogEvent) event); - break; - case LogEvent.ANNOTATE_ROWS_EVENT: - parseAnnotateRowsEvent((AnnotateRowsEvent) event); - break; - case LogEvent.XID_EVENT: - parseXidEvent((XidLogEvent) event); - break; - default: - break; - } + decoder.decode(fecther, context); + continue; + // if (event == null) { + // continue; + // } + // + // int eventType = event.getHeader().getType(); + // switch (eventType) { + // case LogEvent.ROTATE_EVENT: + // binlogFileName = ((RotateLogEvent) event).getFilename(); + // break; + // case LogEvent.WRITE_ROWS_EVENT_V1: + // case LogEvent.WRITE_ROWS_EVENT: + // parseRowsEvent((WriteRowsLogEvent) event); + // break; + // case LogEvent.UPDATE_ROWS_EVENT_V1: + // case LogEvent.UPDATE_ROWS_EVENT: + // parseRowsEvent((UpdateRowsLogEvent) event); + // break; + // case LogEvent.DELETE_ROWS_EVENT_V1: + // case LogEvent.DELETE_ROWS_EVENT: + // parseRowsEvent((DeleteRowsLogEvent) event); + // break; + // case LogEvent.QUERY_EVENT: + // parseQueryEvent((QueryLogEvent) event); + // break; + // case LogEvent.ROWS_QUERY_LOG_EVENT: + // parseRowsQueryEvent((RowsQueryLogEvent) event); + // break; + // case LogEvent.ANNOTATE_ROWS_EVENT: + // parseAnnotateRowsEvent((AnnotateRowsEvent) event); + // break; + // case LogEvent.XID_EVENT: + // parseXidEvent((XidLogEvent) event); + // break; + // default: + // break; + // } } } catch (Exception e) { e.printStackTrace(); diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java index 2ed318e8..f746189e 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java @@ -1,5 +1,6 @@ package com.alibaba.otter.canal.parse.driver.mysql.socket; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -24,7 +25,7 @@ public class BioSocketChannel implements SocketChannel { BioSocketChannel(Socket socket) throws IOException{ this.socket = socket; - this.input = socket.getInputStream(); + this.input = new BufferedInputStream(socket.getInputStream(), 16384); this.output = socket.getOutputStream(); } @@ -164,5 +165,4 @@ public class BioSocketChannel implements SocketChannel { this.socket = null; } - } diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java index dcc17bad..a268ff2e 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java @@ -16,6 +16,8 @@ public abstract class BioSocketChannelPool { socket.setKeepAlive(true); socket.setReuseAddress(true); socket.connect(address, BioSocketChannel.DEFAULT_CONNECT_TIMEOUT); + System.out.println(socket.getReceiveBufferSize()); + System.out.println(socket.getSendBufferSize()); return new BioSocketChannel(socket); } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java index 6849c5b3..e4d81823 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java @@ -41,6 +41,9 @@ public class MysqlDumpTest { controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml"); controller.setEventFilter(new AviaterRegexFilter("test\\..*")); controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*")); + controller.setParallel(true); + controller.setParallelBufferSize(4096); + controller.setParallelThreadSize(16); controller.setIsGTIDMode(false); controller.setEventSink(new AbstractCanalEventSinkTest>() { From 77c1c65b83a13ee416009d028e18555ea4b4a032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 20 Jul 2018 14:32:51 +0800 Subject: [PATCH 09/37] =?UTF-8?q?fixed=20issue=20#726=20,=20=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tddl/dbsync/FetcherPerformanceTest.java | 57 ++++++ .../SimpleCanalClientPermanceTest.java | 68 +++++++ .../parse/MysqlBinlogDumpPerformanceTest.java | 89 ++++++++ .../MysqlBinlogEventPerformanceTest.java | 82 ++++++++ .../MysqlBinlogParsePerformanceTest.java | 192 ++++++++++++++++++ 5 files changed, 488 insertions(+) create mode 100644 dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java new file mode 100644 index 00000000..03fedccd --- /dev/null +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java @@ -0,0 +1,57 @@ +package com.taobao.tddl.dbsync; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.concurrent.atomic.AtomicLong; + +import com.taobao.tddl.dbsync.binlog.DirectLogFetcher; +import com.taobao.tddl.dbsync.binlog.LogContext; +import com.taobao.tddl.dbsync.binlog.LogDecoder; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +public class FetcherPerformanceTest { + + public static void main(String args[]) { + DirectLogFetcher fetcher = new DirectLogFetcher(); + try { + Class.forName("com.mysql.jdbc.Driver"); + Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", + "root", + "hello"); + Statement statement = connection.createStatement(); + statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'"); + statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'"); + + fetcher.open(connection, "mysql-bin.000006", 120L, 2); + + AtomicLong sum = new AtomicLong(0); + long start = System.currentTimeMillis(); + long last = 0; + long end = 0; + + LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); + LogContext context = new LogContext(); + while (fetcher.fetch()) { + decoder.decode(fetcher, context); + sum.incrementAndGet(); + long current = sum.get(); + if (current - last >= 100000) { + end = System.currentTimeMillis(); + long tps = ((current - last) * 1000) / (end - start); + System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps); + last = current; + start = end; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + fetcher.close(); + } catch (IOException e) { + } + } + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java new file mode 100644 index 00000000..c58eb869 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java @@ -0,0 +1,68 @@ +package com.alibaba.otter.canal.example; +import java.net.InetSocketAddress; +import java.util.concurrent.ArrayBlockingQueue; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalConnectors; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.protocol.Message; + +public class SimpleCanalClientPermanceTest { + + public static void main(String args[]) { + String destination = "example"; + String ip = "127.0.0.1"; + int batchSize = 1024; + int count = 0; + int sum = 0; + int perSum = 0; + long start = System.currentTimeMillis(); + long end = 0; + final ArrayBlockingQueue queue = new ArrayBlockingQueue(100); + try { + final CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111), + destination, + "", + ""); + + Thread ackThread = new Thread(new Runnable() { + + @Override + public void run() { + while (true) { + try { + long batchId = queue.take(); + connector.ack(batchId); + } catch (InterruptedException e) { + } + } + } + }); + ackThread.start(); + + ((SimpleCanalConnector) connector).setLazyParseEntry(true); + connector.connect(); + connector.subscribe(); + while (true) { + Message message = connector.getWithoutAck(batchSize); + long batchId = message.getId(); + int size = message.getRawEntries().size(); + sum += size; + perSum += size; + count++; + queue.add(batchId); + if (count % 10 == 0) { + end = System.currentTimeMillis(); + long tps = (perSum * 1000) / (end - start); + System.out.println(" total : " + sum + " , current : " + perSum + " , cost : " + (end - start) + + " , tps : " + tps); + start = end; + perSum = 0; + } + } + } catch (Throwable e) { + e.printStackTrace(); + } + } + +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java new file mode 100644 index 00000000..8ca9b044 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java @@ -0,0 +1,89 @@ +package com.alibaba.otter.canal.parse; + +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.parse.exception.CanalParseException; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; +import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager; +import com.alibaba.otter.canal.parse.support.AuthenticationInfo; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.position.EntryPosition; +import com.alibaba.otter.canal.protocol.position.LogPosition; +import com.alibaba.otter.canal.sink.CanalEventSink; +import com.alibaba.otter.canal.sink.exception.CanalSinkException; + +public class MysqlBinlogDumpPerformanceTest { + + public static void main(String args[]) { + final MysqlEventParser controller = new MysqlEventParser(); + final EntryPosition startPosition = new EntryPosition("mysql-bin.001699", 120L, 100L); + controller.setConnectionCharset(Charset.forName("UTF-8")); + controller.setSlaveId(3344L); + controller.setDetectingEnable(false); + controller.setFilterQueryDml(true); + controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3328), "root", "hello")); + controller.setMasterPosition(startPosition); + controller.setEnableTsdb(false); + controller.setDestination("example"); + controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml"); + // controller.setEventFilter(new AviaterRegexFilter("test\\..*")); + // controller.setEventBlackFilter(new + // AviaterRegexFilter("canal_tsdb\\..*")); + controller.setParallel(true); + controller.setParallelBufferSize(256); + controller.setParallelThreadSize(16); + controller.setIsGTIDMode(false); + final AtomicLong sum = new AtomicLong(0); + final AtomicLong last = new AtomicLong(0); + final AtomicLong start = new AtomicLong(System.currentTimeMillis()); + final AtomicLong end = new AtomicLong(0); + controller.setEventSink(new AbstractCanalEventSinkTest>() { + + public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination) + throws CanalSinkException, + InterruptedException { + + sum.addAndGet(entrys.size()); + long current = sum.get(); + if (current - last.get() >= 100000) { + end.set(System.currentTimeMillis()); + long tps = ((current - last.get()) * 1000) / (end.get() - start.get()); + System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps); + last.set(current); + start.set(end.get()); + } + return true; + } + + }); + controller.setLogPositionManager(new AbstractLogPositionManager() { + + @Override + public LogPosition getLatestIndexBy(String destination) { + return null; + } + + @Override + public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException { + } + }); + + controller.start(); + + try { + Thread.sleep(100 * 1000 * 1000L); + } catch (InterruptedException e) { + } + controller.stop(); + } + + public static abstract class AbstractCanalEventSinkTest extends AbstractCanalLifeCycle implements CanalEventSink { + + public void interrupt() { + } + } +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java new file mode 100644 index 00000000..e92fecb5 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogEventPerformanceTest.java @@ -0,0 +1,82 @@ +package com.alibaba.otter.canal.parse; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector; +import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor; +import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; +import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; +import com.taobao.tddl.dbsync.binlog.LogContext; +import com.taobao.tddl.dbsync.binlog.LogDecoder; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +public class MysqlBinlogEventPerformanceTest { + + protected static Charset charset = Charset.forName("utf-8"); + + public static void main(String args[]) { + DirectLogFetcher fetcher = new DirectLogFetcher(); + try { + MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"); + connector.connect(); + updateSettings(connector); + sendBinlogDump(connector, "mysql-bin.000006", 120L, 3); + fetcher.start(connector.getChannel()); + LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); + LogContext context = new LogContext(); + AtomicLong sum = new AtomicLong(0); + long start = System.currentTimeMillis(); + long last = 0; + long end = 0; + while (fetcher.fetch()) { + decoder.decode(fetcher, context); + sum.incrementAndGet(); + long current = sum.get(); + if (current - last >= 100000) { + end = System.currentTimeMillis(); + long tps = ((current - last) * 1000) / (end - start); + System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps); + last = current; + start = end; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + fetcher.close(); + } catch (IOException e) { + } + } + } + + private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId) + throws IOException { + BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket(); + binlogDumpCmd.binlogFileName = binlogfilename; + binlogDumpCmd.binlogPosition = binlogPosition; + binlogDumpCmd.slaveServerId = slaveId; + byte[] cmdBody = binlogDumpCmd.toBytes(); + + HeaderPacket binlogDumpHeader = new HeaderPacket(); + binlogDumpHeader.setPacketBodyLength(cmdBody.length); + binlogDumpHeader.setPacketSequenceNumber((byte) 0x00); + PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody); + } + + private static void updateSettings(MysqlConnector connector) throws IOException { + update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector); + update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector); + } + + public static void update(String cmd, MysqlConnector connector) throws IOException { + MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector); + exector.update(cmd); + } + +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java new file mode 100644 index 00000000..79b88375 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java @@ -0,0 +1,192 @@ +package com.alibaba.otter.canal.parse; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.util.BitSet; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector; +import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor; +import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; +import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogContext; +import com.taobao.tddl.dbsync.binlog.LogDecoder; +import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer; +import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo; +import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; + +public class MysqlBinlogParsePerformanceTest { + + protected static Charset charset = Charset.forName("utf-8"); + + public static void main(String args[]) { + DirectLogFetcher fetcher = new DirectLogFetcher(); + try { + MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"); + connector.connect(); + updateSettings(connector); + sendBinlogDump(connector, "mysql-bin.000006", 120L, 3); + fetcher.start(connector.getChannel()); + final BlockingQueue buffer = new ArrayBlockingQueue(1024); + Thread thread = new Thread(new Runnable() { + + @Override + public void run() { + try { + consumer(buffer); + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + thread.start(); + + while (fetcher.fetch()) { + buffer.put(fetcher.duplicate()); + fetcher.consume(fetcher.limit()); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + fetcher.close(); + } catch (IOException e) { + } + } + } + + public static void consumer(BlockingQueue buffer) throws IOException, InterruptedException { + LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); + LogContext context = new LogContext(); + + AtomicLong sum = new AtomicLong(0); + long start = System.currentTimeMillis(); + long last = 0; + long end = 0; + while (true) { + LogEvent event = null; + event = decoder.decode(buffer.take(), context); + int eventType = event.getHeader().getType(); + switch (eventType) { + case LogEvent.ROTATE_EVENT: + break; + case LogEvent.WRITE_ROWS_EVENT_V1: + case LogEvent.WRITE_ROWS_EVENT: + parseRowsEvent((WriteRowsLogEvent) event, sum); + break; + case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.UPDATE_ROWS_EVENT: + parseRowsEvent((UpdateRowsLogEvent) event, sum); + break; + case LogEvent.DELETE_ROWS_EVENT_V1: + case LogEvent.DELETE_ROWS_EVENT: + parseRowsEvent((DeleteRowsLogEvent) event, sum); + break; + case LogEvent.XID_EVENT: + sum.incrementAndGet(); + break; + case LogEvent.QUERY_EVENT: + sum.incrementAndGet(); + break; + default: + break; + } + + long current = sum.get(); + if (current - last >= 100000) { + end = System.currentTimeMillis(); + long tps = ((current - last) * 1000) / (end - start); + System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps); + last = current; + start = end; + } + } + } + + private static void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId) + throws IOException { + BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket(); + binlogDumpCmd.binlogFileName = binlogfilename; + binlogDumpCmd.binlogPosition = binlogPosition; + binlogDumpCmd.slaveServerId = slaveId; + byte[] cmdBody = binlogDumpCmd.toBytes(); + + HeaderPacket binlogDumpHeader = new HeaderPacket(); + binlogDumpHeader.setPacketBodyLength(cmdBody.length); + binlogDumpHeader.setPacketSequenceNumber((byte) 0x00); + PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody); + } + + private static void updateSettings(MysqlConnector connector) throws IOException { + update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector); + update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector); + } + + public static void update(String cmd, MysqlConnector connector) throws IOException { + MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector); + exector.update(cmd); + } + + public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) { + try { + RowsLogBuffer buffer = event.getRowsBuf(charset.name()); + BitSet columns = event.getColumns(); + BitSet changeColumns = event.getChangeColumns(); + while (buffer.nextOneRow(columns)) { + int type = event.getHeader().getType(); + if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) { + parseOneRow(event, buffer, columns, true); + } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) { + parseOneRow(event, buffer, columns, false); + } else { + parseOneRow(event, buffer, columns, false); + if (!buffer.nextOneRow(changeColumns)) { + break; + } + parseOneRow(event, buffer, changeColumns, true); + } + + sum.incrementAndGet(); + } + } catch (Exception e) { + throw new RuntimeException("parse row data failed.", e); + } + } + + public static void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter) + throws UnsupportedEncodingException { + TableMapLogEvent map = event.getTable(); + if (map == null) { + throw new RuntimeException("not found TableMap with tid=" + event.getTableId()); + } + + final int columnCnt = map.getColumnCnt(); + final ColumnInfo[] columnInfo = map.getColumnInfo(); + for (int i = 0; i < columnCnt; i++) { + if (!cols.get(i)) { + continue; + } + + ColumnInfo info = columnInfo[i]; + buffer.nextValue(info.type, info.meta); + if (buffer.isNull()) { + } else { + buffer.getValue(); + } + } + } +} From d520260fe891396cc73d0ece48cc662e5c1ca6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 20 Jul 2018 14:46:52 +0800 Subject: [PATCH 10/37] remove unused --- .../java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java index 03fedccd..f5af66e3 100644 --- a/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java @@ -7,8 +7,6 @@ import java.sql.Statement; import java.util.concurrent.atomic.AtomicLong; import com.taobao.tddl.dbsync.binlog.DirectLogFetcher; -import com.taobao.tddl.dbsync.binlog.LogContext; -import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; public class FetcherPerformanceTest { @@ -31,10 +29,7 @@ public class FetcherPerformanceTest { long last = 0; long end = 0; - LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); - LogContext context = new LogContext(); while (fetcher.fetch()) { - decoder.decode(fetcher, context); sum.incrementAndGet(); long current = sum.get(); if (current - last >= 100000) { From 7b52568575b03b0791ef05aadbbf5dff9ee8c93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 20 Jul 2018 16:43:00 +0800 Subject: [PATCH 11/37] fixed unused code --- .../canal/parse/driver/mysql/socket/BioSocketChannelPool.java | 2 -- .../canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java index a268ff2e..dcc17bad 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java @@ -16,8 +16,6 @@ public abstract class BioSocketChannelPool { socket.setKeepAlive(true); socket.setReuseAddress(true); socket.connect(address, BioSocketChannel.DEFAULT_CONNECT_TIMEOUT); - System.out.println(socket.getReceiveBufferSize()); - System.out.println(socket.getSendBufferSize()); return new BioSocketChannel(socket); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java index a85962cc..222d762a 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java @@ -64,9 +64,6 @@ public class DirectLogFetcher extends LogFetcher { if ("1".equals(dbsemi)) { issemi = true; } - // 和mysql driver一样，提供buffer机制，提升读取binlog速度 - // this.input = new - // BufferedInputStream(channel.socket().getInputStream(), 16384); } /** From 89eb3d60785b2b91e42f8984ec8d6a9db3cacbf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 20 Jul 2018 16:45:09 +0800 Subject: [PATCH 12/37] fixed test code --- .../otter/canal/parse/inbound/mysql/MysqlDumpTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java index e4d81823..6c11c637 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java @@ -42,8 +42,8 @@ public class MysqlDumpTest { controller.setEventFilter(new AviaterRegexFilter("test\\..*")); controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*")); controller.setParallel(true); - controller.setParallelBufferSize(4096); - controller.setParallelThreadSize(16); + controller.setParallelBufferSize(256); + controller.setParallelThreadSize(2); controller.setIsGTIDMode(false); controller.setEventSink(new AbstractCanalEventSinkTest>() { From 20807e58c8f189946bdd26a6ea48ac4233f72c18 Mon Sep 17 00:00:00 2001 From: CAI1987 <41417665+CAI1987@users.noreply.github.com> Date: Mon, 23 Jul 2018 15:39:23 +0800 Subject: [PATCH 13/37] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9e596f07..86ef8755 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@
  • canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。
  • canal消费端项目开源: Otter(分布式数据库同步系统)，地址：https://github.com/alibaba/otter
  • Canal已在阿里云推出商业化版本 数据传输服务DTS， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。DTS产品使用文档
    • +DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，立即体验>>>

    背景

    From aaf18ccd7e94150c89e49e2b16f875642818ca29 Mon Sep 17 00:00:00 2001 From: winger Date: Wed, 25 Jul 2018 10:18:15 +0800 Subject: [PATCH 14/37] fix issue #771 #756 --- .../driver/mysql/packets/client/BinlogDumpCommandPacket.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java index 70624549..f0fba11a 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java @@ -54,7 +54,7 @@ public class BinlogDumpCommandPacket extends CommandPacket { // 1. write 4 bytes bin-log position to start at ByteHelper.writeUnsignedIntLittleEndian(binlogPosition, out); // 2. write 2 bytes bin-log flags - int binlog_flags = BINLOG_DUMP_NON_BLOCK; + int binlog_flags = 0; binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT; out.write(binlog_flags); out.write(0x00); From ccd1e52411809a796dccc13fd5b91f3557d9a1e7 Mon Sep 17 00:00:00 2001 From: winger Date: Wed, 25 Jul 2018 16:38:38 +0800 Subject: [PATCH 15/37] fix issue #771 #776 756 --- .../otter/canal/parse/inbound/AbstractEventParser.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index c8cdb1bd..7f7d39ac 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -303,7 +303,12 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle transactionBuffer.reset();// 重置一下缓冲队列，重新记录数据 binlogParser.reset();// 重新置位 if (multiStageCoprocessor != null) { - multiStageCoprocessor.reset(); + // 处理 RejectedExecutionException + try { + multiStageCoprocessor.reset(); + } catch (Throwable t) { + logger.debug("multi processor rejected:", t); + } } if (running) { From e608e1d6012e2cf5bfecaaf068a730633dac8a70 Mon Sep 17 00:00:00 2001 From: winger Date: Thu, 26 Jul 2018 12:33:10 +0800 Subject: [PATCH 16/37] fix bug #776 --- .../com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java index b645e954..5c2678b5 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java @@ -44,13 +44,14 @@ public class GtidLogEvent extends LogEvent { gno = buffer.getLong64(); // support gtid lastCommitted and sequenceNumber - // 42 = 1+16+8+1+8+8 - if (buffer.capacity() > 42 && buffer.getUint8() == LOGICAL_TIMESTAMP_TYPE_CODE) { + // fix bug #776 + if (buffer.hasRemaining() && buffer.remaining() > 16 && buffer.getUint8() == LOGICAL_TIMESTAMP_TYPE_CODE) { lastCommitted = buffer.getLong64(); sequenceNumber = buffer.getLong64(); } + // ignore gtid info read // sid.copy_from((uchar *)ptr_buffer); // ptr_buffer+= ENCODED_SID_LENGTH; From 129051a56e950c84f1cf9673a5e1b6cc3c45538c Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Thu, 26 Jul 2018 14:30:42 +0800 Subject: [PATCH 17/37] Further improvement for issue #771. --- .../mysql/MysqlMultiStageCoprocessor.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 65904b34..fb3539a6 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -4,6 +4,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.LockSupport; +import com.alibaba.otter.canal.common.CanalException; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; @@ -52,17 +53,17 @@ import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; */ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implements MultiStageCoprocessor { - private LogEventConvert logEventConvert; - private EventTransactionBuffer transactionBuffer; - private ErosaConnection connection; + private LogEventConvert logEventConvert; + private EventTransactionBuffer transactionBuffer; + private ErosaConnection connection; - private int parserThreadCount; - private int ringBufferSize; - private RingBuffer disruptorMsgBuffer; - private ExecutorService parserExecutor; - private ExecutorService stageExecutor; - private String destination; - private CanalParseException exception; + private int parserThreadCount; + private int ringBufferSize; + private RingBuffer disruptorMsgBuffer; + private ExecutorService parserExecutor; + private ExecutorService stageExecutor; + private String destination; + private volatile CanalParseException exception; public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert, EventTransactionBuffer transactionBuffer, String destination){ @@ -147,11 +148,14 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement public boolean publish(LogBuffer buffer, String binlogFileName) { if (!isStart()) { - if (exception != null) { - throw exception; - } return false; } + /* 由于改为processor退出不stop，那么需要由exception标识coprocessor是否正常工作。 + * 让dump线程能够及时感知 + */ + if (exception != null) { + throw exception; + } boolean interupted = false; do { @@ -253,9 +257,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement @Override public void onShutdown() { - if (isStart()) { - stop(); - } + } } @@ -291,9 +293,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement @Override public void onShutdown() { - if (isStart()) { - stop(); - } + } } @@ -332,9 +332,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement @Override public void onShutdown() { - if (isStart()) { - stop(); - } + } } @@ -401,7 +399,11 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement @Override public void handleEventException(final Throwable ex, final long sequence, final Object event) { - throw new RuntimeException(ex); + /*如果仅仅抛出去，onShutdown会调用stop()，与instance主parse线程的reset()产生竞争，导致向terminated ExecutorService提交新task + * #771场景中的异常是InterruptedException，由shutdownNow产生。 + */ + exception = new CanalParseException(ex); + throw exception; } @Override From 72b907a75f78b3adae78dba42f16b5c1677cdab9 Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Thu, 26 Jul 2018 14:30:42 +0800 Subject: [PATCH 18/37] Further improvement for issue #771. --- .../mysql/MysqlMultiStageCoprocessor.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index fb3539a6..71dc7ca9 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -64,6 +64,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement private ExecutorService stageExecutor; private String destination; private volatile CanalParseException exception; + private volatile RuntimeException incident; public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert, EventTransactionBuffer transactionBuffer, String destination){ @@ -78,6 +79,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement public void start() { super.start(); this.exception = null; + this.incident = null; this.disruptorMsgBuffer = RingBuffer.createSingleProducer(new MessageEventFactory(), ringBufferSize, new BlockingWaitStrategy()); @@ -148,13 +150,17 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement public boolean publish(LogBuffer buffer, String binlogFileName) { if (!isStart()) { + if (exception != null) { + throw exception; + } return false; } - /* 由于改为processor退出不stop，那么需要由exception标识coprocessor是否正常工作。 + + /* 由于改为processor仅终止自身stage而不是stop，那么需要由incident标识coprocessor是否正常工作。 * 让dump线程能够及时感知 */ - if (exception != null) { - throw exception; + if (incident != null) { + throw incident; } boolean interupted = false; @@ -399,11 +405,12 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement @Override public void handleEventException(final Throwable ex, final long sequence, final Object event) { - /*如果仅仅抛出去，onShutdown会调用stop()，与instance主parse线程的reset()产生竞争，导致向terminated ExecutorService提交新task + /* + * 若异常不是onEvent body里产生的。如果仅仅抛出去，onShutdown会调用stop()，与dump线程的reset()产生竞争，导致向terminated ExecutorService提交新task * #771场景中的异常是InterruptedException，由shutdownNow产生。 */ - exception = new CanalParseException(ex); - throw exception; + incident = new RuntimeException(ex); + throw incident; } @Override From 8cfb70e0809ba12fa5bc9f0131e4412064d8b1ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 26 Jul 2018 16:05:33 +0800 Subject: [PATCH 19/37] =?UTF-8?q?fixed=20issue=20#782=20,=20=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E4=BA=86exception?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/MysqlMultiStageCoprocessor.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 71dc7ca9..281a05f2 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -4,7 +4,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.LockSupport; -import com.alibaba.otter.canal.common.CanalException; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; @@ -64,7 +63,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement private ExecutorService stageExecutor; private String destination; private volatile CanalParseException exception; - private volatile RuntimeException incident; public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert, EventTransactionBuffer transactionBuffer, String destination){ @@ -79,7 +77,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement public void start() { super.start(); this.exception = null; - this.incident = null; this.disruptorMsgBuffer = RingBuffer.createSingleProducer(new MessageEventFactory(), ringBufferSize, new BlockingWaitStrategy()); @@ -156,13 +153,13 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement return false; } - /* 由于改为processor仅终止自身stage而不是stop，那么需要由incident标识coprocessor是否正常工作。 + /** + * 由于改为processor仅终止自身stage而不是stop，那么需要由incident标识coprocessor是否正常工作。 * 让dump线程能够及时感知 */ - if (incident != null) { - throw incident; + if (exception != null) { + throw exception; } - boolean interupted = false; do { try { @@ -180,11 +177,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement interupted = Thread.interrupted(); } } while (!interupted && isStart()); - - if (exception != null) { - throw exception; - } - return isStart(); } @@ -405,12 +397,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement @Override public void handleEventException(final Throwable ex, final long sequence, final Object event) { - /* - * 若异常不是onEvent body里产生的。如果仅仅抛出去，onShutdown会调用stop()，与dump线程的reset()产生竞争，导致向terminated ExecutorService提交新task - * #771场景中的异常是InterruptedException，由shutdownNow产生。 - */ - incident = new RuntimeException(ex); - throw incident; } @Override From c0a40499b0e478398bf838da8b72fa6b2b073bd7 Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Sun, 29 Jul 2018 14:55:38 +0800 Subject: [PATCH 20/37] Canal monitoring(prometheus module) snapshot version. --- deployer/pom.xml | 8 ++ deployer/src/main/bin/metrics_env.sh | 15 +++ deployer/src/main/bin/startup.sh | 7 +- .../example/AbstractCanalClientTest.java | 3 +- pom.xml | 3 +- prometheus/pom.xml | 67 +++++++++++ .../prometheus/CanalInstanceExports.java | 98 ++++++++++++++++ .../canal/prometheus/CanalServerExports.java | 21 ++++ .../canal/prometheus/PrometheusProvider.java | 15 +++ .../canal/prometheus/PrometheusService.java | 110 ++++++++++++++++++ .../impl/InboundThroughputAspect.java | 79 +++++++++++++ .../impl/InstanceMetaCollector.java | 64 ++++++++++ .../prometheus/impl/MemoryStoreCollector.java | 75 ++++++++++++ .../impl/OutboundThroughputAspect.java | 80 +++++++++++++ ...PrometheusCanalEventDownStreamHandler.java | 73 ++++++++++++ .../src/main/resources/META-INF/aop.xml | 11 ++ ...ibaba.otter.canal.spi.CanalMetricsProvider | 1 + .../embedded/CanalServerWithEmbedded.java | 59 +++++++--- .../otter/canal/spi/CanalMetricsProvider.java | 24 ++++ .../otter/canal/spi/CanalMetricsService.java | 42 +++++++ .../canal/spi/NopCanalMetricsService.java | 38 ++++++ 21 files changed, 876 insertions(+), 17 deletions(-) create mode 100644 deployer/src/main/bin/metrics_env.sh create mode 100644 prometheus/pom.xml create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java create mode 100644 prometheus/src/main/resources/META-INF/aop.xml create mode 100644 prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider create mode 100644 server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java diff --git a/deployer/pom.xml b/deployer/pom.xml index 5fa2efc8..75f05805 100644 --- a/deployer/pom.xml +++ b/deployer/pom.xml @@ -16,6 +16,14 @@ canal.server ${project.version} + + + + + + + + diff --git a/deployer/src/main/bin/metrics_env.sh b/deployer/src/main/bin/metrics_env.sh new file mode 100644 index 00000000..1085eae2 --- /dev/null +++ b/deployer/src/main/bin/metrics_env.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Additional line arg for current prometheus solution +case "`uname`" in +Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; +*) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +if [ $(ls $base/lib/aspectjweaver*.jar | wc -l) -eq 1 ]; then + WEAVER=$(ls $base/lib/aspectjweaver*.jar) + METRICS_OPTS=" -javaagent:"${WEAVER}" " +fi diff --git a/deployer/src/main/bin/startup.sh b/deployer/src/main/bin/startup.sh index 5655c713..b87652c0 100644 --- a/deployer/src/main/bin/startup.sh +++ b/deployer/src/main/bin/startup.sh @@ -94,7 +94,12 @@ then echo LOG CONFIGURATION : $logback_configurationFile echo canal conf : $canal_conf echo CLASSPATH :$CLASSPATH - $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & +# metrics support options +# if [ -x $base/bin/metrics_env.sh ]; then +# . $base/bin/metrics_env.sh +# echo METRICS_OPTS $METRICS_OPTS +# fi + $JAVA $JAVA_OPTS $METRICS_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & echo $! > $base/bin/canal.pid echo "cd to $current_path for continue" diff --git a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java index 1b84463b..36979177 100644 --- a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java +++ b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java @@ -3,6 +3,7 @@ package com.alibaba.otter.canal.example; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.SystemUtils; @@ -115,7 +116,7 @@ public class AbstractCanalClientTest { connector.connect(); connector.subscribe(); while (running) { - Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 + Message message = connector.getWithoutAck(batchSize, 1000L, TimeUnit.MILLISECONDS); // 获取指定数量的数据 long batchId = message.getId(); int size = message.getEntries().size(); if (batchId == -1 || size == 0) { diff --git a/pom.xml b/pom.xml index 1e8123ef..8b824ab5 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,7 @@ example kafka kafka-client + prometheus @@ -256,7 +257,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_520 + 2.0.0_preview_186 com.alibaba diff --git a/prometheus/pom.xml b/prometheus/pom.xml new file mode 100644 index 00000000..16cf8acf --- /dev/null +++ b/prometheus/pom.xml @@ -0,0 +1,67 @@ + + + + canal + com.alibaba.otter + 1.0.26-SNAPSHOT + + 4.0.0 + + com.alibaba.otter + canal.prometheus + 1.0.26-SNAPSHOT + canal prometheus module for otter ${project.version} + + + + org.aspectj + aspectjrt + 1.8.9 + + + org.aspectj + aspectjweaver + 1.8.9 + + + org.jctools + jctools-core + 2.1.2 + + + io.prometheus + simpleclient + 0.4.0 + + + io.prometheus + simpleclient_hotspot + 0.4.0 + + + io.prometheus + simpleclient_httpserver + 0.4.0 + + + io.prometheus + simpleclient_pushgateway + 0.4.0 + + + com.alibaba.otter + canal.instance.core + ${project.version} + provided + + + com.alibaba.otter + canal.server + ${project.version} + provided + + + + diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java new file mode 100644 index 00000000..8acf40b8 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -0,0 +1,98 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.prometheus.impl.InstanceMetaCollector; +import com.alibaba.otter.canal.prometheus.impl.MemoryStoreCollector; +import com.alibaba.otter.canal.prometheus.impl.PrometheusCanalEventDownStreamHandler; +import com.alibaba.otter.canal.sink.CanalEventSink; +import com.alibaba.otter.canal.sink.entry.EntryEventSink; +import com.alibaba.otter.canal.store.CanalStoreException; +import io.prometheus.client.Collector; +import io.prometheus.client.CollectorRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; + +/** + * @author Chuanyi Li + */ +public class CanalInstanceExports { + + private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); + + public static final String[] labels = {"destination"}; + + public static final List labelList = Collections.singletonList(labels[0]); + + private final String destination; + + private Collector storeCollector; + + private Collector delayCollector; + + private Collector metaCollector; + + private CanalInstanceExports(CanalInstance instance) { + this.destination = instance.getDestination(); + initDelayGauge(instance); + initStoreCollector(instance); + initMetaCollector(instance); + } + + + + static CanalInstanceExports forInstance(CanalInstance instance) { + return new CanalInstanceExports(instance); + } + + void register() { + if (delayCollector != null) { + delayCollector.register(); + } + if (storeCollector != null) { + storeCollector.register(); + } + if (metaCollector != null) { + metaCollector.register(); + } + } + + void unregister() { + if (delayCollector != null) { + CollectorRegistry.defaultRegistry.unregister(delayCollector); + } + if (storeCollector != null) { + CollectorRegistry.defaultRegistry.unregister(storeCollector); + } + if (metaCollector != null) { + CollectorRegistry.defaultRegistry.unregister(metaCollector); + } + } + + private void initDelayGauge(CanalInstance instance) { + CanalEventSink sink = instance.getEventSink(); + if (sink instanceof EntryEventSink) { + EntryEventSink entryEventSink = (EntryEventSink) sink; + // TODO ensure not to add handler again + PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination); + entryEventSink.addHandler(handler); + delayCollector = handler.getCollector(); + } else { + logger.warn("This impl register metrics for only EntryEventSink, skip."); + } + } + + private void initStoreCollector(CanalInstance instance) { + try { + storeCollector = new MemoryStoreCollector(instance.getEventStore(), destination); + } catch (CanalStoreException cse) { + logger.warn("Failed to register metrics for destination {}.", destination, cse); + } + } + + private void initMetaCollector(CanalInstance instance) { + metaCollector = new InstanceMetaCollector(instance); + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java new file mode 100644 index 00000000..8eef79ba --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java @@ -0,0 +1,21 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.prometheus.impl.InboundThroughputAspect; +import com.alibaba.otter.canal.prometheus.impl.OutboundThroughputAspect; + +/** + * @author Chuanyi Li + */ +public class CanalServerExports { + + private static boolean initialized = false; + + public static synchronized void initialize() { + if (!initialized) { + InboundThroughputAspect.getCollector().register(); + OutboundThroughputAspect.getCollector().register(); + initialized = true; + } + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java new file mode 100644 index 00000000..12b8675f --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java @@ -0,0 +1,15 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.spi.CanalMetricsProvider; +import com.alibaba.otter.canal.spi.CanalMetricsService; + +/** + * @author Chuanyi Li + */ +public class PrometheusProvider implements CanalMetricsProvider { + + @Override + public CanalMetricsService getService() { + return PrometheusService.getInstance(); + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java new file mode 100644 index 00000000..c7942235 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -0,0 +1,110 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.spi.CanalMetricsService; +import io.prometheus.client.exporter.HTTPServer; +import io.prometheus.client.hotspot.DefaultExports; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author Chuanyi Li + */ +public class PrometheusService implements CanalMetricsService { + + private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); + + private final Map exports = new ConcurrentHashMap(); + + private volatile boolean running = false; + + private HTTPServer server; + + private PrometheusService() { + } + + private static class SingletonHolder { + private static final PrometheusService SINGLETON = new PrometheusService(); + } + + public static PrometheusService getInstance() { + return SingletonHolder.SINGLETON; + } + + @Override + public void initialize() { + try { + //TODO 1.Configurable port + //TODO 2.Https + server = new HTTPServer(11112); + } catch (IOException e) { + logger.warn("Unable to start prometheus HTTPServer.", e); + return; + } + try { + // JVM exports + DefaultExports.initialize(); + // Canal server level exports + CanalServerExports.initialize(); + } catch (Throwable t) { + logger.warn("Unable to initialize server exports.", t); + } + + running = true; + } + + @Override + public void terminate() { + running = false; + // Normally, service should be terminated at canal shutdown. + // No need to unregister instance exports explicitly. + // But for the sake of safety, unregister them. + for (CanalInstanceExports ie : exports.values()) { + ie.unregister(); + } + if (server != null) { + server.stop(); + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void register(CanalInstance instance) { + if (instance.isStart()) { + logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination()); + return; + } + try { + CanalInstanceExports export = CanalInstanceExports.forInstance(instance); + export.register(); + exports.put(instance.getDestination(), export); + } catch (Throwable t) { + logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t); + } + logger.info("Register metrics for destination {}.", instance.getDestination()); + } + + @Override + public void unregister(CanalInstance instance) { + if (instance.isStart()) { + logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination()); + } + try { + CanalInstanceExports export = exports.remove(instance.getDestination()); + if (export != null) { + export.unregister(); + } + } catch (Throwable t) { + logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t); + } + logger.info("Unregister metrics for destination {}.", instance.getDestination()); + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java new file mode 100644 index 00000000..eabc256c --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java @@ -0,0 +1,79 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import org.aspectj.lang.annotation.After; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.jctools.maps.ConcurrentAutoTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Chuanyi Li + */ +@Aspect +public class InboundThroughputAspect { + + private static final Logger logger = LoggerFactory.getLogger(InboundThroughputAspect.class); + + /** + * Support highly scalable counters + * @see ConcurrentAutoTable + */ + private static final ConcurrentAutoTable total = new ConcurrentAutoTable(); + + private static final Collector collector = new InboundThroughputCollector(); + + public static Collector getCollector() { + return collector; + } + + @Pointcut("call(byte[] com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))") + public void read() {} + + @Pointcut("call(void com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..)) ") + public void readBytes() {} + + //nested read, just eliminate them. + @Pointcut("withincode(* com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))") + public void nestedCall() {} + + @After("read() && !nestedCall() && args(len, ..)") + public void recordRead(int len) { + accumulateBytes(len); + } + + @After("readBytes() && !nestedCall() && args(.., len, timeout)") + public void recordReadBytes(int len, int timeout) { + accumulateBytes(len); + } + + private void accumulateBytes(int count) { + try { + total.add(count); + } catch (Throwable t) { + //Catch every Throwable, rather than break the business logic. + logger.warn("Error while accumulate inbound bytes.", t); + } + } + + public static class InboundThroughputCollector extends Collector { + + private InboundThroughputCollector() {} + + @Override + public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily bytes = new CounterMetricFamily("canal_net_inbound_bytes", + "Total socket inbound bytes of canal server.", + total.get()); + mfs.add(bytes); + return mfs; + } + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java new file mode 100644 index 00000000..5562b8fd --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java @@ -0,0 +1,64 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring; +import com.alibaba.otter.canal.meta.CanalMetaManager; +import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.protocol.ClientIdentity; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author Chuanyi Li + */ +public class InstanceMetaCollector extends Collector { + + private static final List InfoLabel = Arrays.asList("destination", "mode"); + + private CanalMetaManager metaManager; + + private final String destination; + + private final String mode; + + private final String subsHelp; + + public InstanceMetaCollector(CanalInstance instance) { + if (instance == null) { + throw new IllegalArgumentException("CanalInstance must not be null."); + } + if (instance instanceof CanalInstanceWithSpring) { + mode = "spring"; + } else { + mode = "manager"; + } + this.metaManager = instance.getMetaManager(); + this.destination = instance.getDestination(); + this.subsHelp = "Subscriptions of canal instance " + destination; + } + + @Override + public List collect() { + List mfs = new ArrayList(); + GaugeMetricFamily instanceInfo = new GaugeMetricFamily( + "canal_instance", + "Canal instance", + InfoLabel); + instanceInfo.addMetric(Arrays.asList(destination, mode), 1); + mfs.add(instanceInfo); + if (metaManager.isStart()) { + // client id = hardcode 1001, 目前没有意义 + List subs = metaManager.listAllSubscribeInfo(destination); + GaugeMetricFamily subscriptions = new GaugeMetricFamily( + "canal_instance_subscription", + subsHelp, CanalInstanceExports.labelList); + subscriptions.addMetric(Arrays.asList(destination), subs.size()); + mfs.add(subscriptions); + } + return mfs; + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java new file mode 100644 index 00000000..79fdca4a --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java @@ -0,0 +1,75 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.store.CanalEventStore; +import com.alibaba.otter.canal.store.CanalStoreException; +import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Chuanyi Li + */ +public class MemoryStoreCollector extends Collector { + + private static final Class clazz = MemoryEventStoreWithBuffer.class; + + private final String destination; + + private final AtomicLong putSequence; + + private final AtomicLong ackSequence; + + private final String putHelp; + + private final String ackHelp; + + public MemoryStoreCollector(CanalEventStore store, String destination) { + this.destination = destination; + if (!(store instanceof MemoryEventStoreWithBuffer)) { + throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer"); + } + MemoryEventStoreWithBuffer ms = (MemoryEventStoreWithBuffer) store; + putSequence = getDeclaredValue(ms, "putSequence"); + ackSequence = getDeclaredValue(ms, "ackSequence"); + putHelp = "Produced sequence of canal instance " + destination; + ackHelp = "Consumed sequence of canal instance " + destination; + } + + @Override + public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily put = new CounterMetricFamily("canal_instance_store_produce_seq", + putHelp, Arrays.asList(CanalInstanceExports.labels)); + put.addMetric(Collections.singletonList(destination), putSequence.doubleValue()); + mfs.add(put); + CounterMetricFamily ack = new CounterMetricFamily("canal_instance_store_consume_seq", + ackHelp, Arrays.asList(CanalInstanceExports.labels)); + ack.addMetric(Collections.singletonList(destination), ackSequence.doubleValue()); + mfs.add(ack); + return mfs; + } + + @SuppressWarnings("unchecked") + private static T getDeclaredValue(MemoryEventStoreWithBuffer store, String name) { + T value; + try { + Field putField = clazz.getDeclaredField(name); + putField.setAccessible(true); + value = (T) putField.get(store); + } catch (NoSuchFieldException e) { + throw new CanalStoreException(e); + } catch (IllegalAccessException e) { + throw new CanalStoreException(e); + } + return value; + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java new file mode 100644 index 00000000..60dd808d --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java @@ -0,0 +1,80 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import org.aspectj.lang.annotation.After; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.jboss.netty.channel.Channel; +import org.jctools.maps.ConcurrentAutoTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static com.alibaba.otter.canal.server.netty.NettyUtils.HEADER_LENGTH; + +/** + * @author Chuanyi Li + */ +@Aspect +public class OutboundThroughputAspect { + private static final Logger logger = LoggerFactory.getLogger(OutboundThroughputAspect.class); + + /** + * Support highly scalable counters + * @see ConcurrentAutoTable + */ + private static final ConcurrentAutoTable total = new ConcurrentAutoTable(); + + private static final Collector collector = new OutboundThroughputCollector(); + + public static Collector getCollector() { + return collector; + } + + @Pointcut("call(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))") + public void write() {} + + //nested read, just eliminate them. + @Pointcut("withincode(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))") + public void nestedCall() {} + + @After("write() && !nestedCall() && args(ch, bytes, ..)") + public void recordWriteBytes(Channel ch, byte[] bytes) { + if (bytes != null) { + accumulateBytes(HEADER_LENGTH + bytes.length); + } + } + + @After("write() && !nestedCall() && args(ch, buf, ..)") + public void recordWriteBuffer(Channel ch, ByteBuffer buf) { + if (buf != null) { + total.add(HEADER_LENGTH + buf.limit()); + } + } + private void accumulateBytes(int count) { + try { + total.add(count); + } catch (Throwable t) { + //Catch every Throwable, rather than break the business logic. + logger.warn("Error while accumulate inbound bytes.", t); + } + } + + public static class OutboundThroughputCollector extends Collector { + + private OutboundThroughputCollector() {} + + @Override public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily bytes = new CounterMetricFamily("canal_net_outbound_bytes", + "Total socket outbound bytes of canal server.", + total.get()); + mfs.add(bytes); + return mfs; + } + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java new file mode 100644 index 00000000..c55a7eac --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java @@ -0,0 +1,73 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler; +import com.alibaba.otter.canal.store.model.Event; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * @author Chuanyi Li + */ +public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler> { + + private final Collector collector; + + private long latestExecuteTime = 0L; + + private static final String DELAY_NAME = "canal_instance_traffic_delay"; + + private final String delayHelpName; + + private final List labelValues; + + public PrometheusCanalEventDownStreamHandler(final String destination) { + this.delayHelpName = "Traffic delay of canal instance " + destination + " in seconds."; + this.labelValues = Collections.singletonList(destination); + collector = new Collector() { + @Override + public List collect() { + List mfs = new ArrayList(); + long now = System.currentTimeMillis(); + GaugeMetricFamily delay = new GaugeMetricFamily( + DELAY_NAME, + delayHelpName, + CanalInstanceExports.labelList); + double d = 0.0; + if (latestExecuteTime > 0) { + d = now - latestExecuteTime; + } + d = d > 0.0 ? (d / 1000) : 0.0; + delay.addMetric(labelValues, d); + mfs.add(delay); + return mfs; + } + }; + } + + @Override + public List before(List events) { + // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming + // see: https://dev.mysql.com/worklog/task/?id=342 + // heartbeats are sent by the master only if there is no + // more unsent events in the actual binlog file for a period longer that + // master_heartbeat_period. + if (events != null && !events.isEmpty()) { + Event last = events.get(events.size() - 1); + long ts = last.getExecuteTime(); + if (ts > latestExecuteTime) { + latestExecuteTime = ts; + } + } + return events; + } + + public Collector getCollector() { + return this.collector; + } + +} diff --git a/prometheus/src/main/resources/META-INF/aop.xml b/prometheus/src/main/resources/META-INF/aop.xml new file mode 100644 index 00000000..8bd31d0c --- /dev/null +++ b/prometheus/src/main/resources/META-INF/aop.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider b/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider new file mode 100644 index 00000000..9adbf8d1 --- /dev/null +++ b/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider @@ -0,0 +1 @@ +com.alibaba.otter.canal.prometheus.PrometheusProvider diff --git a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java index 722bb2bc..ccf21020 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java @@ -1,11 +1,11 @@ package com.alibaba.otter.canal.server.embedded; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.spi.CanalMetricsProvider; +import com.alibaba.otter.canal.spi.CanalMetricsService; +import com.alibaba.otter.canal.spi.NopCanalMetricsService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -33,17 +33,18 @@ import com.google.protobuf.ByteString; /** * 嵌入式版本实现 - * + * * @author jianghang 2012-7-12 下午01:34:00 * @author zebin.xuzb * @version 1.0.0 */ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, CanalService { - private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class); + private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class); private Map canalInstances; // private Map lastRollbackPostions; private CanalInstanceGenerator canalInstanceGenerator; + private CanalMetricsService metrics = NopCanalMetricsService.NOP; private static class SingletonHolder { @@ -61,7 +62,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C public void start() { if (!isStart()) { super.start(); - + // 如果存在provider,则启动metrics service + loadCanalMetrics(); + metrics.initialize(); canalInstances = MigrateMap.makeComputingMap(new Function() { public CanalInstance apply(String destination) { @@ -92,6 +95,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e); } } + metrics.terminate(); } public void start(final String destination) { @@ -99,6 +103,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C if (!canalInstance.isStart()) { try { MDC.put("destination", destination); + if (metrics.isRunning()) { + metrics.register(canalInstance); + } canalInstance.start(); logger.info("start CanalInstances[{}] successfully", destination); } finally { @@ -114,6 +121,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C try { MDC.put("destination", destination); canalInstance.stop(); + if (metrics.isRunning()) { + metrics.unregister(canalInstance); + } logger.info("stop CanalInstances[{}] successfully", destination); } finally { MDC.remove("destination"); @@ -176,7 +186,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 获取数据 - * + * * 
          * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
      *
    @@ -188,14 +198,14 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 获取数据，可以指定超时时间. - * + * * 
          * 几种case:
      * a. 如果timeout为null，则采用tryGet方式，即时获取
      * b. 如果timeout不为null
      *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
      *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
-     * 
+     *
      * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
      *
    */ @@ -251,7 +261,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 不指定 position 获取事件。canal 会记住此 client 最新的 position。
    * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。 - * + * * 
          * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
      *
    @@ -264,14 +274,14 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 不指定 position 获取事件。canal 会记住此 client 最新的 position。
    * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。 - * + * * 
          * 几种case:
      * a. 如果timeout为null，则采用tryGet方式，即时获取
      * b. 如果timeout不为null
      *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
      *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
-     *    
+     *
      * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
      *
    */ @@ -342,7 +352,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。 - * + * * 
          * 注意：进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
      *
    @@ -492,6 +502,27 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } } + private void loadCanalMetrics() { + ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class); + List list = new ArrayList(); + for (CanalMetricsProvider provider : providers) { + list.add(provider); + } + if (!list.isEmpty()) { + // 发现provider, 进行初始化 + if (list.size() > 1) { + logger.warn("Found more than one CanalMetricsProvider, use the first one."); + //报告冲突 + for (CanalMetricsProvider p : list) { + logger.warn("Found CanalMetricsProvider: {}.", p.getClass().getName()); + } + } + //默认使用第一个 + CanalMetricsProvider provider = list.get(0); + this.metrics = provider.getService(); + } + } + // ========= setter ========== public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) { diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java new file mode 100644 index 00000000..ad0ceddf --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java @@ -0,0 +1,24 @@ +package com.alibaba.otter.canal.spi; + +/** + * Use java service provider mechanism to provide {@link CanalMetricsService}. + * 
    + * Example:
+ * {@code
+ *     ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class);
+ *     List list = new ArrayList();
+ *     for (CanalMetricsProvider provider : providers) {
+ *         list.add(provider);
+ *     }
+ * }
+ *
    + * @author Chuanyi Li + */ +public interface CanalMetricsProvider { + + /** + * @return Impl of {@link CanalMetricsService} + */ + CanalMetricsService getService(); + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java new file mode 100644 index 00000000..6d823be8 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java @@ -0,0 +1,42 @@ +package com.alibaba.otter.canal.spi; + +import com.alibaba.otter.canal.instance.core.CanalInstance; + +/** + * Canal server/instance metrics for export. + * + * Designed to be created by service provider. + * + * @see CanalMetricsProvider + * @author Chuanyi Li + */ +public interface CanalMetricsService { + + /** + * Initialization on canal server startup. + */ + void initialize(); + + /** + * Clean-up at canal server stop phase. + */ + void terminate(); + + /** + * @return {@code true} if the metrics service is running, otherwise {@code false}. + */ + boolean isRunning(); + + /** + * Register instance level metrics for specified instance. + * @param instance {@link CanalInstance} + */ + void register(CanalInstance instance); + + /** + * Unregister instance level metrics for specified instance. + * @param instance {@link CanalInstance} + */ + void unregister(CanalInstance instance); + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java b/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java new file mode 100644 index 00000000..0d6ad088 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.spi; + +import com.alibaba.otter.canal.instance.core.CanalInstance; + +/** + * @author Chuanyi Li + */ +public class NopCanalMetricsService implements CanalMetricsService { + + public static final NopCanalMetricsService NOP = new NopCanalMetricsService(); + + private NopCanalMetricsService() {} + + @Override + public void initialize() { + + } + + @Override + public void terminate() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public void register(CanalInstance instance) { + + } + + @Override + public void unregister(CanalInstance instance) { + + } +} From 1eaf198acfc0ae4464446315aba773ca50a27eff Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Sun, 29 Jul 2018 15:06:30 +0800 Subject: [PATCH 21/37] Revert test code. --- .../alibaba/otter/canal/example/AbstractCanalClientTest.java | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java index 36979177..d1ddd88e 100644 --- a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java +++ b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java @@ -116,7 +116,7 @@ public class AbstractCanalClientTest { connector.connect(); connector.subscribe(); while (running) { - Message message = connector.getWithoutAck(batchSize, 1000L, TimeUnit.MILLISECONDS); // 获取指定数量的数据 + Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 long batchId = message.getId(); int size = message.getEntries().size(); if (batchId == -1 || size == 0) { diff --git a/pom.xml b/pom.xml index 8b824ab5..59c5bc01 100644 --- a/pom.xml +++ b/pom.xml @@ -257,7 +257,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_186 + 2.0.0_preview_520 com.alibaba From 3d5f13fec8b840f5d913103264db8f7f4c2c1f72 Mon Sep 17 00:00:00 2001 From: winger Date: Tue, 31 Jul 2018 10:34:19 +0800 Subject: [PATCH 22/37] =?UTF-8?q?kafka=20producer=20=E9=80=82=E9=85=8D=20r?= =?UTF-8?q?ow=20data=20for=20performance=20#726?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kafka/producer/CanalKafkaProducer.java | 27 ++++++++++++++----- .../kafka/producer/MessageSerializer.java | 25 +++++++++-------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java index 1543abf9..3318a89e 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java @@ -1,7 +1,9 @@ package com.alibaba.otter.canal.kafka.producer; +import java.io.IOException; import java.util.Properties; +import com.google.protobuf.ByteString; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -49,14 +51,25 @@ public class CanalKafkaProducer { } } - public void send(Topic topic, Message message) { + public void send(Topic topic, Message message) throws IOException { boolean valid = false; - if (message != null && !message.getEntries().isEmpty()) { - for (CanalEntry.Entry entry : message.getEntries()) { - if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN - && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { - valid = true; - break; + if (message != null) { + if (message.isRaw() && !message.getRawEntries().isEmpty()) { + for (ByteString byteString : message.getRawEntries()) { + CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); + if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN + && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { + valid = true; + break; + } + } + } else if (!message.getEntries().isEmpty()){ + for (CanalEntry.Entry entry : message.getEntries()) { + if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN + && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { + valid = true; + break; + } } } } diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java index 339e4f8c..17fca117 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java @@ -1,14 +1,13 @@ package com.alibaba.otter.canal.kafka.producer; -import java.util.Map; - -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Serializer; -import org.springframework.util.CollectionUtils; - import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalPacket; import com.alibaba.otter.canal.protocol.Message; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Serializer; +import org.springframework.util.CollectionUtils; + +import java.util.Map; /** * Kafka Message类的序列化 @@ -25,12 +24,15 @@ public class MessageSerializer implements Serializer { @Override public byte[] serialize(String topic, Message data) { try { - if (data == null) return null; - else { + if (data != null) { CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); - if (data.getId() != -1 && !CollectionUtils.isEmpty(data.getEntries())) { - for (CanalEntry.Entry entry : data.getEntries()) { - messageBuilder.addMessages(entry.toByteString()); + if (data.getId() != -1) { + if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) { + messageBuilder.addAllMessages(data.getRawEntries()); + } else if (!CollectionUtils.isEmpty(data.getEntries())) { + for (CanalEntry.Entry entry : data.getEntries()) { + messageBuilder.addMessages(entry.toByteString()); + } } } CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); @@ -41,6 +43,7 @@ public class MessageSerializer implements Serializer { } catch (Exception e) { throw new SerializationException("Error when serializing message to byte[] "); } + return null; } @Override From 93ebc7b973b282c38d9cfacd7d7846cac309403b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 1 Aug 2018 18:25:49 +0800 Subject: [PATCH 23/37] fixed unused repo --- pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index 59c5bc01..9a14868b 100644 --- a/pom.xml +++ b/pom.xml @@ -66,16 +66,6 @@ false - - alibaba - http://code.alibabatech.com/mvn/releases/ - - true - - - false - - sonatype sonatype From 26edbeaa280b8c3617fea51f16f9358d67f659cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 2 Aug 2018 13:58:41 +0800 Subject: [PATCH 24/37] fixed kafka performance --- deployer/src/main/resources/canal.properties | 1 + .../resources/spring/default-instance.xml | 1 + .../main/resources/spring/file-instance.xml | 1 + .../main/resources/spring/group-instance.xml | 1 + .../main/resources/spring/local-instance.xml | 1 + .../main/resources/spring/memory-instance.xml | 1 + .../kafka/client/MessageDeserializer.java | 4 +- .../client/running/ClientRunningData.java | 4 +- .../client/running/ClientRunningMonitor.java | 82 ++++++++++--------- .../otter/canal/kafka/CanalServerStarter.java | 6 +- .../kafka/producer/CanalKafkaProducer.java | 51 ++++++------ .../kafka/producer/CanalKafkaStarter.java | 5 +- .../canal/kafka/producer/KafkaProperties.java | 25 ++++-- .../kafka/producer/MessageSerializer.java | 59 ++++++++++--- kafka/src/main/resources/kafka.yml | 1 + 15 files changed, 154 insertions(+), 89 deletions(-) diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 23ec03a1..0a11a5c9 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -43,6 +43,7 @@ canal.instance.filter.query.dml = false canal.instance.filter.query.ddl = false canal.instance.filter.table.error = false canal.instance.filter.rows = false +canal.instance.filter.transaction.entry = false # binlog format/image check canal.instance.binlog.format = ROW,STATEMENT,MIXED diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index c4465df1..9bf28896 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -81,6 +81,7 @@ + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 340a8158..6d263717 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -67,6 +67,7 @@ + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 60d069dc..60ceff01 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -64,6 +64,7 @@ + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index 0b2f96aa..2020d0d6 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -67,6 +67,7 @@ + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index 8fec8918..c08f3f12 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -64,6 +64,7 @@ + diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java index 1cb3daba..70f2fc06 100644 --- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java +++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java @@ -29,7 +29,9 @@ public class MessageDeserializer implements Deserializer { @Override public Message deserialize(String topic, byte[] data) { try { - if (data == null) return null; + if (data == null) { + return null; + } else { CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data); switch (p.getType()) { diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java index 3eab85ed..7bbbabd7 100644 --- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java +++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java @@ -8,8 +8,8 @@ package com.alibaba.otter.canal.kafka.client.running; */ public class ClientRunningData { - private String groupId; - private String address; + private String groupId; + private String address; private boolean active = true; public String getGroupId() { diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java index 3bdbfe6e..0f61211e 100644 --- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java +++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java @@ -1,12 +1,11 @@ package com.alibaba.otter.canal.kafka.client.running; -import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; -import com.alibaba.otter.canal.common.utils.AddressUtils; -import com.alibaba.otter.canal.common.utils.BooleanMutex; -import com.alibaba.otter.canal.common.utils.JsonUtils; -import com.alibaba.otter.canal.common.zookeeper.ZkClientx; -import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; -import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import java.text.MessageFormat; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + import org.I0Itec.zkclient.IZkDataListener; import org.I0Itec.zkclient.exception.ZkException; import org.I0Itec.zkclient.exception.ZkInterruptedException; @@ -17,11 +16,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import java.text.MessageFormat; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.utils.BooleanMutex; +import com.alibaba.otter.canal.common.utils.JsonUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + /** * kafka client running状态信息 @@ -31,13 +33,18 @@ import java.util.concurrent.TimeUnit; */ public class ClientRunningMonitor extends AbstractCanalLifeCycle { - private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics"; + private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE + + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics"; - private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "{0}"; + private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + + "{0}"; - private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "{1}"; + private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + + "{1}"; - private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + ZookeeperPathUtils.RUNNING_NODE; + private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE + + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + + ZookeeperPathUtils.RUNNING_NODE; private static String getTopicClientRunning(String topic, String groupId) { return MessageFormat.format(TOPIC_CLIENTID_RUNNING_NODE, topic, groupId); @@ -47,21 +54,21 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { return MessageFormat.format(TOPIC_CLIENTID_NODE, topic, groupId); } - private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class); - private ZkClientx zkClient; - private String topic; - private ClientRunningData clientData; - private IZkDataListener dataListener; - private BooleanMutex mutex = new BooleanMutex(false); - private volatile boolean release = false; + private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class); + private ZkClientx zkClient; + private String topic; + private ClientRunningData clientData; + private IZkDataListener dataListener; + private BooleanMutex mutex = new BooleanMutex(false); + private volatile boolean release = false; private volatile ClientRunningData activeData; - private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); - private ClientRunningListener listener; - private int delayTime = 5; + private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); + private ClientRunningListener listener; + private int delayTime = 5; - private static Integer virtualPort; + private static Integer virtualPort; - public ClientRunningMonitor() { + public ClientRunningMonitor(){ if (virtualPort == null) { Random rand = new Random(); virtualPort = rand.nextInt(9000) + 1000; @@ -108,7 +115,6 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { } - public void start() { super.start(); @@ -123,7 +129,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { String path = getTopicClientRunning(this.topic, clientData.getGroupId()); zkClient.unsubscribeDataChanges(path, dataListener); releaseRunning(); // 尝试一下release - //Fix issue #697 + // Fix issue #697 if (delayExector != null) { delayExector.shutdown(); } @@ -159,13 +165,12 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { } } } catch (ZkNoNodeException e) { - zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), - true); // 尝试创建父节点 + zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), true); // 尝试创建父节点 initRunning(); } catch (Throwable t) { logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].", - topic), - t); + topic), + t); // 出现任何异常尝试release releaseRunning(); throw new CanalClientException("something goes wrong in initRunning method. ", t); @@ -187,7 +192,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { */ public boolean check() { String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - //ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + // ZookeeperPathUtils.getDestinationClientRunning(this.destination, + // clientData.getClientId()); try { byte[] bytes = zkClient.readData(path); ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class); @@ -196,8 +202,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { boolean result = isMine(activeData.getAddress()); if (!result) { logger.warn("canal is running in [{}] , but not in [{}]", - activeData.getAddress(), - clientData.getAddress()); + activeData.getAddress(), + clientData.getAddress()); } return result; } catch (ZkNoNodeException e) { @@ -235,7 +241,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { if (listener != null) { // 触发回调 listener.processActiveEnter(); - this.clientData.setAddress(/*address*/AddressUtils.getHostIp() + ":" + virtualPort); + this.clientData.setAddress(/* address */AddressUtils.getHostIp() + ":" + virtualPort); String path = getTopicClientRunning(this.topic, clientData.getGroupId()); // 序列化 diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java index e0b2d1d8..573f0eac 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java @@ -17,9 +17,9 @@ import com.alibaba.otter.canal.deployer.CanalController; */ public class CanalServerStarter { - private static final String CLASSPATH_URL_PREFIX = "classpath:"; - private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class); - private volatile static boolean running = false; + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class); + private volatile static boolean running = false; public static void init() { try { diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java index 3318a89e..4543aac1 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java @@ -3,7 +3,6 @@ package com.alibaba.otter.canal.kafka.producer; import java.io.IOException; import java.util.Properties; -import com.google.protobuf.ByteString; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -12,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic; -import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.Message; /** @@ -52,31 +50,32 @@ public class CanalKafkaProducer { } public void send(Topic topic, Message message) throws IOException { - boolean valid = false; - if (message != null) { - if (message.isRaw() && !message.getRawEntries().isEmpty()) { - for (ByteString byteString : message.getRawEntries()) { - CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); - if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN - && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { - valid = true; - break; - } - } - } else if (!message.getEntries().isEmpty()){ - for (CanalEntry.Entry entry : message.getEntries()) { - if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN - && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { - valid = true; - break; - } - } - } - } - if (!valid) { - return; - } + // set canal.instance.filter.transaction.entry = true + // boolean valid = false; + // if (message != null) { + // if (message.isRaw() && !message.getRawEntries().isEmpty()) { + // for (ByteString byteString : message.getRawEntries()) { + // CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); + // if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN + // && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { + // valid = true; + // break; + // } + // } + // } else if (!message.getEntries().isEmpty()){ + // for (CanalEntry.Entry entry : message.getEntries()) { + // if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN + // && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { + // valid = true; + // break; + // } + // } + // } + // } + // if (!valid) { + // return; + // } ProducerRecord record; if (topic.getPartition() != null) { record = new ProducerRecord(topic.getTopic(), topic.getPartition(), null, message); diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java index 0ff018c2..e88f9354 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java @@ -52,7 +52,10 @@ public class CanalKafkaStarter { // 初始化 kafka producer canalKafkaProducer = new CanalKafkaProducer(); canalKafkaProducer.init(kafkaProperties); - + // set filterTransactionEntry + if (kafkaProperties.isFilterTransactionEntry()) { + System.setProperty("canal.instance.filter.transaction.entry", "true"); + } // 对应每个instance启动一个worker线程 List destinations = kafkaProperties.getCanalDestinations(); diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java index 0b346137..8e193aac 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java @@ -13,15 +13,15 @@ import java.util.Set; */ public class KafkaProperties { - private String servers = "localhost:6667"; - private int retries = 0; - private int batchSize = 16384; - private int lingerMs = 1; - private long bufferMemory = 33554432L; + private String servers = "localhost:6667"; + private int retries = 0; + private int batchSize = 16384; + private int lingerMs = 1; + private long bufferMemory = 33554432L; + private boolean filterTransactionEntry = true; + private int canalBatchSize = 5; - private int canalBatchSize = 5; - - private List canalDestinations = new ArrayList(); + private List canalDestinations = new ArrayList(); public static class CanalDestination { @@ -158,4 +158,13 @@ public class KafkaProperties { public void setCanalDestinations(List canalDestinations) { this.canalDestinations = canalDestinations; } + + public boolean isFilterTransactionEntry() { + return filterTransactionEntry; + } + + public void setFilterTransactionEntry(boolean filterTransactionEntry) { + this.filterTransactionEntry = filterTransactionEntry; + } + } diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java index 17fca117..02336004 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java @@ -1,13 +1,19 @@ package com.alibaba.otter.canal.kafka.producer; -import com.alibaba.otter.canal.protocol.CanalEntry; -import com.alibaba.otter.canal.protocol.CanalPacket; -import com.alibaba.otter.canal.protocol.Message; +import java.util.List; +import java.util.Map; + import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serializer; import org.springframework.util.CollectionUtils; -import java.util.Map; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.CanalPacket; +import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; +import com.alibaba.otter.canal.protocol.Message; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.WireFormat; /** * Kafka Message类的序列化 @@ -25,20 +31,53 @@ public class MessageSerializer implements Serializer { public byte[] serialize(String topic, Message data) { try { if (data != null) { - CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); if (data.getId() != -1) { if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) { - messageBuilder.addAllMessages(data.getRawEntries()); + // for performance + List rowEntries = data.getRawEntries(); + // message size + int messageSize = 0; + messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, data.getId()); + + int dataSize = 0; + for (int i = 0; i < rowEntries.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i)); + } + messageSize += dataSize; + messageSize += 1 * rowEntries.size(); + // packet size + int size = 0; + size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, + PacketType.MESSAGES.getNumber()); + size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + + messageSize; + // build data + byte[] body = new byte[size]; + CodedOutputStream output = CodedOutputStream.newInstance(body); + output.writeEnum(3, PacketType.MESSAGES.getNumber()); + + output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(messageSize); + // message + output.writeInt64(1, data.getId()); + for (int i = 0; i < rowEntries.size(); i++) { + output.writeBytes(2, rowEntries.get(i)); + } + output.checkNoSpaceLeft(); + return body; } else if (!CollectionUtils.isEmpty(data.getEntries())) { + CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); for (CanalEntry.Entry entry : data.getEntries()) { messageBuilder.addMessages(entry.toByteString()); } + + CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); + packetBuilder.setType(CanalPacket.PacketType.MESSAGES); + packetBuilder.setBody(messageBuilder.build().toByteString()); + return packetBuilder.build().toByteArray(); } } - CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); - packetBuilder.setType(CanalPacket.PacketType.MESSAGES); - packetBuilder.setBody(messageBuilder.build().toByteString()); - return packetBuilder.build().toByteArray(); } } catch (Exception e) { throw new SerializationException("Error when serializing message to byte[] "); diff --git a/kafka/src/main/resources/kafka.yml b/kafka/src/main/resources/kafka.yml index ab214319..d6ccc51a 100644 --- a/kafka/src/main/resources/kafka.yml +++ b/kafka/src/main/resources/kafka.yml @@ -5,6 +5,7 @@ lingerMs: 1 bufferMemory: 33554432 # canal的批次大小，单位 k canalBatchSize: 50 +filterTransactionEntry: true canalDestinations: - canalDestination: example From da1ce51a0ff52ee289ce96243bc4c5952b2dd7b6 Mon Sep 17 00:00:00 2001 From: winger Date: Fri, 3 Aug 2018 11:11:38 +0800 Subject: [PATCH 25/37] =?UTF-8?q?=E6=94=AF=E6=8C=81GTID=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E9=80=9A=E8=BF=87=E8=8E=B7=E5=8F=96=E5=BD=93?= =?UTF-8?q?=E5=89=8Dgtid=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/parse/inbound/mysql/MysqlEventParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 6c2b9d4d..e640364f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -643,6 +643,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation"); } EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1))); + if (isGTIDMode && fields.size() > 4) { + endPosition.setGtid(fields.get(4)); + } return endPosition; } catch (IOException e) { throw new CanalParseException("command : 'show master status' has an error!", e); From 7fd958a490d9a5bb1d42d43c84ddbc5e6bf4d6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 3 Aug 2018 17:48:32 +0800 Subject: [PATCH 26/37] fixed issue #801 , support canal docker --- .gitignore | 2 + .../otter/canal/deployer/CanalController.java | 13 +- deployer/src/main/resources/canal.properties | 9 +- .../resources/example/instance.properties | 19 ++- .../resources/spring/default-instance.xml | 4 +- .../main/resources/spring/file-instance.xml | 4 +- .../main/resources/spring/group-instance.xml | 8 +- .../main/resources/spring/local-instance.xml | 2 +- .../main/resources/spring/memory-instance.xml | 4 +- docker/Dockerfile | 69 +++++++++++ docker/build.sh | 30 +++++ docker/image/admin/app.sh | 117 ++++++++++++++++++ docker/image/admin/bin/clean_log | 2 + docker/image/admin/bin/clean_log.sh | 45 +++++++ docker/image/admin/health.sh | 13 ++ docker/image/alidata/bin/exec_rc_local.sh | 11 ++ docker/image/alidata/bin/lark-wait | 6 + docker/image/alidata/bin/main.sh | 27 ++++ docker/image/alidata/init/02init-sshd.sh | 19 +++ docker/image/alidata/init/fix-hosts.py | 66 ++++++++++ docker/image/alidata/lib/proc.sh | 40 ++++++ docker/run.sh | 92 ++++++++++++++ .../parse/driver/mysql/MysqlConnector.java | 2 +- 23 files changed, 580 insertions(+), 24 deletions(-) create mode 100644 docker/Dockerfile create mode 100644 docker/build.sh create mode 100755 docker/image/admin/app.sh create mode 100644 docker/image/admin/bin/clean_log create mode 100755 docker/image/admin/bin/clean_log.sh create mode 100755 docker/image/admin/health.sh create mode 100755 docker/image/alidata/bin/exec_rc_local.sh create mode 100755 docker/image/alidata/bin/lark-wait create mode 100755 docker/image/alidata/bin/main.sh create mode 100755 docker/image/alidata/init/02init-sshd.sh create mode 100755 docker/image/alidata/init/fix-hosts.py create mode 100644 docker/image/alidata/lib/proc.sh create mode 100644 docker/run.sh diff --git a/.gitignore b/.gitignore index 3f1a5d6b..90bd720f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ jtester.properties .idea/ *.iml .DS_Store +*.tar.gz +*.rpm diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java index 88f6e9a3..0a2c91f8 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java @@ -379,7 +379,18 @@ public class CanalController { } private String getProperty(Properties properties, String key) { - return StringUtils.trim(properties.getProperty(StringUtils.trim(key))); + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); } public void start() throws Throwable { diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 0a11a5c9..2bff29ae 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -3,7 +3,7 @@ ################################################# canal.id= 1 canal.ip= -canal.port= 11111 +canal.port=11111 canal.zkServers= # flush data to zk canal.zookeeper.flush.period = 1000 @@ -59,6 +59,13 @@ canal.instance.parser.parallel = true ## disruptor ringbuffer size, must be power of 2 canal.instance.parser.parallelBufferSize = 256 +# table meta tsdb info +canal.instance.tsdb.enable=true +canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +canal.instance.tsdb.dbUsername=canal +canal.instance.tsdb.dbPassword=canal + ################################################# ######### destinations ############# ################################################# diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index a3703b86..e0cf0064 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -1,11 +1,12 @@ ################################################# -## mysql serverId -canal.instance.mysql.slaveId=0 +## mysql serverId , v1.0.26+ will autoGen +# canal.instance.mysql.slaveId=0 + +# enable gtid use true/false +canal.instance.gtidon=false # position info canal.instance.master.address=127.0.0.1:3306 -# enable gtid use true/false -canal.instance.gtidon=false canal.instance.master.journal.name= canal.instance.master.position= canal.instance.master.timestamp= @@ -13,23 +14,21 @@ canal.instance.master.gtid= # table meta tsdb info canal.instance.tsdb.enable=true -canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} -canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb -canal.instance.tsdb.dbUsername=canal -canal.instance.tsdb.dbPassword=canal - +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal #canal.instance.standby.address = #canal.instance.standby.journal.name = #canal.instance.standby.position = #canal.instance.standby.timestamp = #canal.instance.standby.gtid= + # username/password canal.instance.dbUsername=canal canal.instance.dbPassword=canal -canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 + # table regex canal.instance.filter.regex=.*\\..* # table black regex diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index 9bf28896..6b455c83 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -149,7 +149,7 @@ - + @@ -157,7 +157,7 @@ - + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 6d263717..377428a9 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -134,7 +134,7 @@ - + @@ -142,7 +142,7 @@ - + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 60ceff01..44f1ab3b 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -131,7 +131,7 @@ - + @@ -139,7 +139,7 @@ - + @@ -229,7 +229,7 @@ - + @@ -237,7 +237,7 @@ - + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index 2020d0d6..930ecb7b 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -114,7 +114,7 @@ - + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index c08f3f12..cd387b8f 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -122,7 +122,7 @@ - + @@ -130,7 +130,7 @@ - + diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..7e10085b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,69 @@ +FROM centos:centos6.7 + +MAINTAINER agapple (jianghang115@gmail.com) + +# install system +RUN \ + /bin/cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo 'root:Hello1234' | chpasswd && \ + groupadd -r admin && useradd -g admin admin && \ + yum install -y man && \ + yum install -y dstat && \ + yum install -y unzip && \ + yum install -y nc && \ + yum install -y openssh-server && \ + yum install -y tar && \ + yum install -y which && \ + yum install -y wget && \ + yum install -y perl && \ + yum install -y file && \ + ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \ + ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \ + sed -ri 's/session required pam_loginuid.so/#session required pam_loginuid.so/g' /etc/pam.d/sshd && \ + sed -i -e 's/^#Port 22$/Port 2222/' /etc/ssh/sshd_config && \ + mkdir -p /root/.ssh && chown root.root /root && chmod 700 /root/.ssh && \ + yum install -y cronie && \ + sed -i '/session required pam_loginuid.so/d' /etc/pam.d/crond && \ + yum clean all && \ + true + +# install canal +COPY image/ /tmp/docker/ +COPY canal.deployer-*.tar.gz /home/admin/ +COPY jdk-8-linux-x64.rpm /tmp/ + +RUN \ + cp -R /tmp/docker/alidata /alidata && \ + chmod +x /alidata/bin/* && \ + mkdir -p /home/admin && \ + cp -R /tmp/docker/admin/* /home/admin/ && \ + /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \ + + touch /var/lib/rpm/* && \ + yum -y install /tmp/jdk-8-linux-x64.rpm && \ + /bin/rm -f /tmp/jdk-8-linux-x64.rpm && \ + + echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ + echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ + /bin/mv /home/admin/bin/canal-check /etc/cron.d && \ + /bin/mv /home/admin/bin/clean_log /etc/cron.d && \ + chmod +x /home/admin/bin/canal-process-check.py && \ + + mkdir -p /home/admin/canal-server && \ + tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ + /bin/rm -f /home/admin/canal.deployer-*.tar.gz && \ + + mkdir -p home/admin/canal-server/logs && \ + chmod +x /home/admin/*.sh && \ + chmod +x /home/admin/bin/*.sh && \ + chown admin: -R /home/admin && \ + yum clean all && \ + true + +# 2222 sys , 8080 web , 8000 debug , 11111 canal +EXPOSE 2222 11111 8000 8080 + +WORKDIR /home/admin + +ENTRYPOINT [ "/alidata/bin/main.sh" ] +CMD [ "/home/admin/app.sh" ] \ No newline at end of file diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 00000000..58424fb3 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} + +if [ ! -f $BASE/jdk*.rpm ] ; then + DOWNLOAD_LINK="http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz" + wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=xxx; oraclelicense=accept-securebackup-cookie" "$DOWNLOAD_LINK" -O $BASE/jdk-8-linux-x64.rpm +fi + +cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ; + +if [ "$1" == "kafka" ] ; then + cp $BASE/../target/canal-kafka-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-server $BASE/ +else + cp $BASE/../target/canal.deployer-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-server $BASE/ +fi diff --git a/docker/image/admin/app.sh b/docker/image/admin/app.sh new file mode 100755 index 00000000..254f7e48 --- /dev/null +++ b/docker/image/admin/app.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e + +source /etc/profile +export JAVA_HOME=/usr/java/latest +export PATH=$JAVA_HOME/bin:$PATH +touch /tmp/start.log +chown admin: /tmp/start.log +chown -R admin: /home/admin/canal-server +host=`hostname -i` + +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} + + +function checkStart() { + local name=$1 + local cmd=$2 + local timeout=$3 + cost=5 + while [ $timeout -gt 0 ]; do + ST=`eval $cmd` + if [ "$ST" == "0" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + elif [ "$ST" == "" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + else + break + fi + done + echo "start $name successful" +} + + +function start_canal() { + echo "start canal ..." + serverPort=`perl -le 'print $ENV{"canal.port"}'` + if [ -z "$serverPort" ] ; then + serverPort=11111 + fi + + destination=`perl -le 'print $ENV{"canal.destinations"}'` + if [[ "$destination" =~ ',' ]]; then + echo "multi destination:$destination is not support" + exit 1; + else + mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination + fi + su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1' + sleep 5 + #check start + checkStart "canal" "nc 127.0.0.1 $serverPort -w 1 -z | wc -l" 30 +} + +function stop_canal() { + echo "stop canal" + su admin -c 'cd /home/admin/canal-server/bin/ && sh stop.sh 1>>/tmp/start.log 2>&1' + echo "stop canal successful ..." +} + +echo "==> START ..." + +start_canal + +echo "==> START SUCCESSFUL ..." + +tail -f /dev/null & +# wait TERM signal +waitterm + +echo "==> STOP" + +stop_canal + +echo "==> STOP SUCCESSFUL ..." diff --git a/docker/image/admin/bin/clean_log b/docker/image/admin/bin/clean_log new file mode 100644 index 00000000..df183614 --- /dev/null +++ b/docker/image/admin/bin/clean_log @@ -0,0 +1,2 @@ +# cron clean log once per minute +*/2 * * * * admin /home/admin/bin/clean_log.sh >>/tmp/clean_log.log 2>&1 diff --git a/docker/image/admin/bin/clean_log.sh b/docker/image/admin/bin/clean_log.sh new file mode 100755 index 00000000..9d0a0bef --- /dev/null +++ b/docker/image/admin/bin/clean_log.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Global Settings +PATH="$HOME/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/X11R6/bin:/root/bin" +export PATH + +CUTOFF="85" +#获取磁盘使用率最高的分区 +USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) +before=$USAGE + +baseClean(){ + #删除tmp目录15天前的文件。 + #更新文档时间戳 + if [ -d /tmp/hsperfdata_admin ] + then + touch /tmp/hsperfdata_admin + touch /tmp/hsperfdata_admin/* + fi + + find /tmp/ -type f -mtime +15 | xargs -t rm -rf >/dev/null 2>&1 + + + now=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + echo "before:$before; now:$now" +} + +CANAL_DIR="/home/admin/canal-server/logs" +if [[ -d $CANAL_DIR ]]; then + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 90 ]]; then + find $CANAL_DIR -type f -mtime +7 | xargs rm -rf {} + fi + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 80 ]]; then + find $CANAL_DIR -type f -mtime +3 | xargs rm -rf {} + fi + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 80 ]]; then + find $CANAL_DIR -type d -empty -mtime +3 | grep -v canal | xargs rm -rf {} + find $CANAL_DIR -type f -iname '*.tmp' | xargs rm -rf {} + fi + baseClean + exit 0 +fi \ No newline at end of file diff --git a/docker/image/admin/health.sh b/docker/image/admin/health.sh new file mode 100755 index 00000000..17a89740 --- /dev/null +++ b/docker/image/admin/health.sh @@ -0,0 +1,13 @@ +#!/bin/sh +CHECK_URL="http://127.0.0.1:8080/metrics" +CHECK_POINT="success" +CHECK_COUNT=`curl -s --connect-timeout 7 --max-time 7 $CHECK_URL | grep -c $CHECK_POINT` +if [ $CHECK_COUNT -eq 0 ]; then + echo "[FAILED]" + status=0 + error=1 +else + echo "[ OK ]" + status=1 + error=0 +fi \ No newline at end of file diff --git a/docker/image/alidata/bin/exec_rc_local.sh b/docker/image/alidata/bin/exec_rc_local.sh new file mode 100755 index 00000000..7cc8aa3f --- /dev/null +++ b/docker/image/alidata/bin/exec_rc_local.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ "${SKIP_EXEC_RC_LOCAL}" = "YES" ] ; then + echo "skip /etc/rc.local: SKIP_EXEC_RC_LOCAL=${SKIP_EXEC_RC_LOCAL}" + exit +fi + +if [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] ; then + echo "skip /etc/rc.local: DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" + exit +fi \ No newline at end of file diff --git a/docker/image/alidata/bin/lark-wait b/docker/image/alidata/bin/lark-wait new file mode 100755 index 00000000..db7be708 --- /dev/null +++ b/docker/image/alidata/bin/lark-wait @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +chown admin: -R /home/admin/ +source /alidata/lib/proc.sh +waitterm diff --git a/docker/image/alidata/bin/main.sh b/docker/image/alidata/bin/main.sh new file mode 100755 index 00000000..a57551a7 --- /dev/null +++ b/docker/image/alidata/bin/main.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +[ -n "${DOCKER_DEPLOY_TYPE}" ] || DOCKER_DEPLOY_TYPE="VM" +echo "DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" + +# run init scripts +for e in $(ls /alidata/init/*) ; do + [ -x "${e}" ] || continue + echo "==> INIT $e" + $e + echo "==> EXIT CODE: $?" +done + +echo "==> INIT DEFAULT" +service sshd start +service crond start + +#echo "check hostname -i: `hostname -i`" +#hti_num=`hostname -i|awk '{print NF}'` +#if [ $hti_num -gt 1 ];then +# echo "hostname -i result error:`hostname -i`" +# exit 120 +#fi + +echo "==> INIT DONE" +echo "==> RUN ${*}" +exec "${@}" \ No newline at end of file diff --git a/docker/image/alidata/init/02init-sshd.sh b/docker/image/alidata/init/02init-sshd.sh new file mode 100755 index 00000000..118ee553 --- /dev/null +++ b/docker/image/alidata/init/02init-sshd.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# set port +if [ -z "${SSHD_PORT}" ] ; then + SSHD_PORT=22 + [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] && SSHD_PORT=2222 +fi + +sed -r -i '/^OPTIONS=/ d' /etc/sysconfig/sshd +echo 'OPTIONS="-p '"${SSHD_PORT}"'"' >> /etc/sysconfig/sshd + +# set admin ssh pulic key +if [ "${USE_ADMIN_PASSAGE}" = "YES" ] ; then + echo "set admin passage" + mkdir -p /home/admin/.ssh + chown admin:admin /home/admin/.ssh + chown admin:admin /home/admin/.ssh/authorized_keys + chmod 644 /home/admin/.ssh/authorized_keys +fi diff --git a/docker/image/alidata/init/fix-hosts.py b/docker/image/alidata/init/fix-hosts.py new file mode 100755 index 00000000..4ee8abb5 --- /dev/null +++ b/docker/image/alidata/init/fix-hosts.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +#****************************************************************# +# Create Date: 2017-01-06 17:58 +#***************************************************************# + +import socket +import shutil +from time import gmtime, strftime + +# get host_name +host_name = socket.gethostname() +tmp_file = "/tmp/.lark-fix-host.hosts" +host_file = "/etc/hosts" +bak_file_name = "/tmp/hosts-fix-bak.%s" % ( strftime("%Y-%m-%d_%H-%M-%S", gmtime()) ) + +# load /etc/hosts file context +FH = open(host_file,"r") +file_lines = [ i.rstrip() for i in FH.readlines()] +FH.close() +file_lines_reverse = file_lines[::-1] +new_lines = [] +bad_lines = [] +last_match_line = "" + +for line in file_lines_reverse: + if line.find(host_name) < 0: # 不匹配的行直接跳过 + new_lines.append(line + "\n") + continue + + cols = line.split() + new_cols = [] + if cols[0].startswith("#"): # 跳过已经注释掉的行 + new_lines.append(line + "\n") + continue + for col in cols: + if not col == host_name: # 跳过不匹配的列 + new_cols.append(col) + continue + + if cols[0] == "127.0.0.1": # 如果第一列是 127.0.0.1 就跳过匹配的列, 防止 hostname -i 返回 127.0.0.1 + continue + + # 如果已经发现过匹配的列, 就丢掉重复的列 + if not len(last_match_line) == 0: + continue + + new_cols.append(col) + last_match_line = line + + # 跳过 xx.xx.xx.xx hostname 这样的重复列 + if len(new_cols) == 1: + continue + + new_l = "%s\n" % " ".join(new_cols) + new_lines.append(new_l) + +# save tmp hosts + +FH2=file(tmp_file,"w+") +FH2.writelines( new_lines[::-1]) +FH2.close() + +# mv to /etc/hosts +shutil.copy(host_file, bak_file_name) +shutil.move(tmp_file, host_file) diff --git a/docker/image/alidata/lib/proc.sh b/docker/image/alidata/lib/proc.sh new file mode 100644 index 00000000..40ddff93 --- /dev/null +++ b/docker/image/alidata/lib/proc.sh @@ -0,0 +1,40 @@ +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} diff --git a/docker/run.sh b/docker/run.sh new file mode 100644 index 00000000..7c56e2ba --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +function usage() { + echo "Usage:" + echo " run.sh [CONFIG]" + echo "example:" + echo " run.sh -e canal.instance.master.address=127.0.0.1:3306 \\" + echo " -e canal.instance.dbUsername=canal \\" + echo " -e canal.instance.dbPassword=canal \\" + echo " -e canal.instance.connectionCharset=UTF-8 \\" + echo " -e canal.instance.tsdb.enable=true \\" + echo " -e canal.instance.gtidon=false \\" + echo " -e canal.instance.filter.regex=.*\\..* " + exit +} + +function check_port() { + local port=$1 + local TL=$(which telnet) + if [ -f $TL ]; then + data=`echo quit | telnet 127.0.0.1 $port| grep -ic connected` + echo $data + return + fi + + local NC=$(which nc) + if [ -f $NC ]; then + data=`nc -z -w 1 127.0.0.1 $port | grep -ic succeeded` + echo $data + return + fi + echo "0" + return +} + +function getMyIp() { + case "`uname`" in + Darwin) + myip=`echo "show State:/Network/Global/IPv4" | scutil | grep PrimaryInterface | awk '{print $3}' | xargs ifconfig | grep inet | grep -v inet6 | awk '{print $2}'` + ;; + *) + myip=`ip route get 1 | awk '{print $NF;exit}'` + ;; + esac + echo $myip +} + +NET_MODE="" +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + NET_MODE="--net=host" + ;; + *) + NET_MODE="--net=host" + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} +if [ $# -eq 0 ]; then + usage +elif [ "$1" == "-h" ] ; then + usage +elif [ "$1" == "help" ] ; then + usage +fi + +DATA="$BASE/data" +mkdir -p $DATA +CONFIG=${@:1} +#VOLUMNS="-v $DATA:/home/admin/canal-server/logs" +PORTLIST="8000 8080 2222 11111" +PORTS="" +for PORT in $PORTLIST ; do + #exist=`check_port $PORT` + exist="0" + if [ "$exist" == "0" ]; then + PORTS="$PORTS -p $PORT:$PORT" + else + echo "port $PORT is used , pls check" + exit 1 + fi +done + +MEMORY="-m 4096m" +LOCALHOST=`getMyIp` +cmd="docker run -it -h $LOCALHOST $CONFIG --name=canal-server $VOLUMNS $NET_MODE $PORTS $MEMORY canal/canal-server" +echo $cmd +eval $cmd \ No newline at end of file diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 29b96852..0f01cbd5 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -32,7 +32,7 @@ public class MysqlConnector { private String password; private byte charsetNumber = 33; - private String defaultSchema = "retl"; + private String defaultSchema = "test"; private int soTimeout = 30 * 1000; private int connTimeout = 5 * 1000; private int receiveBufferSize = 16 * 1024; From 872dad8ce82e21e3acfdb5d3f596e81e52b44f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Sat, 4 Aug 2018 10:37:55 +0800 Subject: [PATCH 27/37] fixed issue #801 , optimizer docker --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e10085b..e44796cb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,9 +45,7 @@ RUN \ echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ - /bin/mv /home/admin/bin/canal-check /etc/cron.d && \ /bin/mv /home/admin/bin/clean_log /etc/cron.d && \ - chmod +x /home/admin/bin/canal-process-check.py && \ mkdir -p /home/admin/canal-server && \ tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ From 3c716e2a436f729f1394e9a92e86be2e9095da1e Mon Sep 17 00:00:00 2001 From: qmzhang <578967933@qq.com> Date: Sun, 5 Aug 2018 19:40:14 +0800 Subject: [PATCH 28/37] =?UTF-8?q?instance=E7=94=B1manager=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E6=97=B6=EF=BC=8CCanalParameter#MetaMode=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E6=9C=AC=E5=9C=B0meta.dat=E5=AD=98=E5=82=A8=E4=BD=8D?= =?UTF-8?q?=E7=82=B9=E4=BF=A1=E6=81=AF=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/CanalInstanceWithManager.java | 21 +++----- .../manager/model/CanalParameter.java | 26 ++++++++- .../CanalServerWithEmbedded_FileModeTest.java | 53 +++++++++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index bca5ba13..35e3235a 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import com.alibaba.otter.canal.meta.FileMixedMetaManager; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,13 +21,7 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.instance.core.AbstractCanalInstance; import com.alibaba.otter.canal.instance.manager.model.Canal; import com.alibaba.otter.canal.instance.manager.model.CanalParameter; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; import com.alibaba.otter.canal.meta.MemoryMetaManager; import com.alibaba.otter.canal.meta.PeriodMixedMetaManager; import com.alibaba.otter.canal.meta.ZooKeeperMetaManager; @@ -37,12 +32,7 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; -import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager; -import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager; -import com.alibaba.otter.canal.parse.index.MetaLogPositionManager; -import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager; -import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager; +import com.alibaba.otter.canal.parse.index.*; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.sink.entry.EntryEventSink; @@ -120,6 +110,11 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(getZkclientx()); ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager); + } else if (mode.isLocalFile()){ + FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager(); + fileMixedMetaManager.setDataDir(parameters.getDataDir()); + fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod()); + metaManager = fileMixedMetaManager; } else { throw new CanalException("unsupport MetaMode for " + mode); } diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index 16c929ca..befc8e2f 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -28,8 +28,10 @@ public class CanalParameter implements Serializable { private Long zkClusterId; // zk集群id，为管理方便 private List zkClusters; // zk集群地址 + private String dataDir = "../conf"; // 默认本地文件数据的目录默认是conf // meta相关参数 private MetaMode metaMode = MetaMode.MEMORY; // meta机制 + private Integer metaFileFlushPeriod = 1000; // meta刷新间隔 // storage存储 private Integer transactionSize = 1024; // 支持处理的transaction事务大小 @@ -243,7 +245,9 @@ public class CanalParameter implements Serializable { /** 文件存储模式 */ ZOOKEEPER, /** 混合模式，内存+文件 */ - MIXED; + MIXED, + /** 本地文件存储模式*/ + LOCAL_FILE; public boolean isMemory() { return this.equals(MetaMode.MEMORY); @@ -256,6 +260,10 @@ public class CanalParameter implements Serializable { public boolean isMixed() { return this.equals(MetaMode.MIXED); } + + public boolean isLocalFile(){ + return this.equals(MetaMode.LOCAL_FILE); + } } public static enum IndexMode { @@ -390,6 +398,22 @@ public class CanalParameter implements Serializable { return storageMode; } + public String getDataDir() { + return dataDir; + } + + public void setDataDir(String dataDir) { + this.dataDir = dataDir; + } + + public Integer getMetaFileFlushPeriod() { + return metaFileFlushPeriod; + } + + public void setMetaFileFlushPeriod(Integer metaFileFlushPeriod) { + this.metaFileFlushPeriod = metaFileFlushPeriod; + } + public void setStorageMode(StorageMode storageMode) { this.storageMode = storageMode; } diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java new file mode 100644 index 00000000..e986b6ff --- /dev/null +++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java @@ -0,0 +1,53 @@ +package com.alibaba.otter.canal.server; + +import java.net.InetSocketAddress; +import java.util.Arrays; + +import com.alibaba.otter.canal.instance.manager.model.Canal; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; + +public class CanalServerWithEmbedded_FileModeTest extends BaseCanalServerWithEmbededTest { + + protected Canal buildCanal() { + Canal canal = new Canal(); + canal.setId(1L); + canal.setName(DESTINATION); + canal.setDesc("my standalone server test "); + + CanalParameter parameter = new CanalParameter(); + + parameter.setMetaMode(MetaMode.LOCAL_FILE); + parameter.setDataDir("./conf"); + parameter.setMetaFileFlushPeriod(1000); + parameter.setHaMode(HAMode.HEARTBEAT); + parameter.setIndexMode(IndexMode.MEMORY_META_FAILBACK); + + parameter.setStorageMode(StorageMode.MEMORY); + parameter.setMemoryStorageBufferSize(32 * 1024); + + parameter.setSourcingType(SourcingType.MYSQL); + parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306), + new InetSocketAddress(MYSQL_ADDRESS, 3306))); + parameter.setDbUsername(USERNAME); + parameter.setDbPassword(PASSWORD); + parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}", + "{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}")); + + parameter.setSlaveId(1234L); + + parameter.setDefaultConnectionTimeoutInSeconds(30); + parameter.setConnectionCharset("UTF-8"); + parameter.setConnectionCharsetNumber((byte) 33); + parameter.setReceiveBufferSize(8 * 1024); + parameter.setSendBufferSize(8 * 1024); + + parameter.setDetectingEnable(false); + parameter.setDetectingIntervalInSeconds(10); + parameter.setDetectingRetryTimes(3); + parameter.setDetectingSQL(DETECTING_SQL); + + canal.setCanalParameter(parameter); + return canal; + } +} \ No newline at end of file From 8a952d3a5f41939b249bc85e7cb3b81ee41ff873 Mon Sep 17 00:00:00 2001 From: winger Date: Mon, 6 Aug 2018 11:11:57 +0800 Subject: [PATCH 29/37] fix bug: kafka get row data for performance --- .../alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java index e88f9354..0eedfa7a 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java @@ -121,7 +121,7 @@ public class CanalKafkaStarter { Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据 long batchId = message.getId(); try { - int size = message.getEntries().size(); + int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); if (batchId != -1 && size != 0) { if (!StringUtils.isEmpty(destination.getTopic())) { Topic topic = new Topic(); From 7d9e2dc69b63df1053c5ee89541291e4f9b114ef Mon Sep 17 00:00:00 2001 From: winger Date: Tue, 7 Aug 2018 12:04:57 +0800 Subject: [PATCH 30/37] =?UTF-8?q?fix=20#802:=20=E4=BB=8Eotter=E4=B8=AD?= =?UTF-8?q?=E5=89=A5=E7=A6=BBMySQL=20=E7=9B=B8=E5=85=B3=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=EF=BC=8C=E4=BD=9C=E4=B8=BAexample=E7=9A=84?= =?UTF-8?q?=E5=8F=82=E8=80=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- example/pom.xml | 81 +++ .../canal/example/db/AbstractDbClient.java | 144 ++++++ .../example/db/CanalConnectorClient.java | 488 ++++++++++++++++++ .../canal/example/db/MysqlLoadLauncher.java | 35 ++ .../canal/example/db/ServiceLocator.java | 44 ++ .../example/db/dialect/AbstractDbDialect.java | 121 +++++ .../db/dialect/AbstractSqlTemplate.java | 105 ++++ .../canal/example/db/dialect/DbDialect.java | 20 + .../canal/example/db/dialect/SqlTemplate.java | 40 ++ .../canal/example/db/dialect/TableType.java | 93 ++++ .../db/dialect/mysql/MysqlDialect.java | 32 ++ .../db/dialect/mysql/MysqlSqlTemplate.java | 84 +++ .../example/db/mysql/AbstractMysqlClient.java | 207 ++++++++ .../canal/example/db/mysql/MysqlClient.java | 23 + .../example/db/utils/ByteArrayConverter.java | 50 ++ .../canal/example/db/utils/DdlUtils.java | 326 ++++++++++++ .../db/utils/SqlTimestampConverter.java | 140 +++++ .../canal/example/db/utils/SqlUtils.java | 315 +++++++++++ example/src/main/resources/client-spring.xml | 53 ++ example/src/main/resources/client.properties | 16 + 20 files changed, 2417 insertions(+) create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/AbstractDbClient.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/MysqlLoadLauncher.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/ServiceLocator.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractDbDialect.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractSqlTemplate.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/DbDialect.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/SqlTemplate.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/TableType.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlDialect.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlSqlTemplate.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/mysql/AbstractMysqlClient.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/mysql/MysqlClient.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/utils/ByteArrayConverter.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/utils/DdlUtils.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlTimestampConverter.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlUtils.java create mode 100644 example/src/main/resources/client-spring.xml create mode 100644 example/src/main/resources/client.properties diff --git a/example/pom.xml b/example/pom.xml index 1f60bc01..ba7a6dde 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -21,6 +21,79 @@ canal.protocol ${project.version} + + com.alibaba.otter + canal.instance.spring + ${project.version} + + + org.apache.ddlutils + ddlutils + 1.0 + + + commons-beanutils + commons-beanutils-core + + + commons-lang + commons-lang + + + commons-dbcp + commons-dbcp + + + commons-pool + commons-pool + + + commons-logging + commons-logging-api + + + dom4j + dom4j + + + stax + stax-api + + + commons-collections + commons-collections + + + commons-digester + commons-digester + + + commons-betwixt + commons-betwixt + + + + + org.apache.commons + commons-pool2 + 2.5.0 + + + commons-beanutils + commons-beanutils + 1.8.2 + + + org.apache.commons + commons-lang3 + 3.7 + + + commons-collections + commons-collections + 3.2 + + junit @@ -62,6 +135,14 @@ false + + org.apache.maven.plugins + maven-compiler-plugin + + 7 + 7 + +     diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/AbstractDbClient.java b/example/src/main/java/com/alibaba/otter/canal/example/db/AbstractDbClient.java new file mode 100644 index 00000000..ec998e55 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/AbstractDbClient.java @@ -0,0 +1,144 @@ +package com.alibaba.otter.canal.example.db; + +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.Message; +import org.slf4j.MDC; + +import java.util.Date; +import java.util.List; + +public abstract class AbstractDbClient extends CanalConnectorClient { + + + public abstract void insert(CanalEntry.Header header, List afterColumns); + + public abstract void update(CanalEntry.Header header, List afterColumns); + + public abstract void delete(CanalEntry.Header header, List beforeColumns); + + + @Override + public synchronized void start() { + if (running) { + return; + } + super.start(); + } + + @Override + public synchronized void stop() { + if (!running) { + return; + } + super.stop(); + MDC.remove("destination"); + } + + @Override + protected void processMessage(Message message) { + long batchId = message.getId(); + //遍历每条消息 + for (CanalEntry.Entry entry : message.getEntries()) { + session(entry);//no exception + } + //ack all the time。 + connector.ack(batchId); + } + + private void session(CanalEntry.Entry entry) { + CanalEntry.EntryType entryType = entry.getEntryType(); + int times = 0; + boolean success = false; + while (!success) { + if (times > 0) { + /** + * 1:retry，重试，重试默认为3次，由retryTimes参数决定，如果重试次数达到阈值，则跳过，并且记录日志。 + * 2:ignore,直接忽略，不重试，记录日志。 + */ + if (exceptionStrategy == ExceptionStrategy.RETRY.code) { + if (times >= retryTimes) { + break; + } + } else { + break; + } + } + try { + switch (entryType) { + case TRANSACTIONBEGIN: + transactionBegin(entry); + break; + case TRANSACTIONEND: + transactionEnd(entry); + break; + case ROWDATA: + rowData(entry); + break; + default: + break; + } + success = true; + } catch (Exception e) { + times++; + logger.error("parse event has an error ,times: + " + times + ", data:" + entry.toString(), e); + } + + } + } + + private void rowData(CanalEntry.Entry entry) throws Exception { + CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); + CanalEntry.EventType eventType = rowChange.getEventType(); + CanalEntry.Header header = entry.getHeader(); + long executeTime = header.getExecuteTime(); + long delayTime = new Date().getTime() - executeTime; + String sql = rowChange.getSql(); + + try { + if (!isDML(eventType) || rowChange.getIsDdl()) { + processDDL(header, eventType, sql); + return; + } + //处理DML数据 + processDML(header, eventType, rowChange, sql); + } catch (Exception e) { + logger.error("process event error ,", e); + logger.error(rowFormat, + new Object[]{header.getLogfileName(), String.valueOf(header.getLogfileOffset()), + header.getSchemaName(), header.getTableName(), eventType, + String.valueOf(executeTime), String.valueOf(delayTime)}); + throw e;//重新抛出 + } + } + + /** + * 处理 dml 数据 + * + * @param header + * @param eventType + * @param rowChange + * @param sql + */ + protected void processDML(CanalEntry.Header header, CanalEntry.EventType eventType, CanalEntry.RowChange rowChange, String sql) { + for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) { + switch (eventType) { + case DELETE: + delete(header, rowData.getBeforeColumnsList()); + break; + case INSERT: + insert(header, rowData.getAfterColumnsList()); + break; + case UPDATE: + update(header, rowData.getAfterColumnsList()); + break; + default: + whenOthers(header, sql); + } + } + } + +} + + + + diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java b/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java new file mode 100644 index 00000000..844f60db --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/CanalConnectorClient.java @@ -0,0 +1,488 @@ +package com.alibaba.otter.canal.example.db; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalConnectors; +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.Message; +import org.apache.commons.lang.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.util.CollectionUtils; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; + +public abstract class CanalConnectorClient extends AbstractCanalLifeCycle implements InitializingBean { + + protected static final Logger logger = LoggerFactory.getLogger(CanalConnectorClient.class); + protected static final String SEP = SystemUtils.LINE_SEPARATOR; + protected static String contextFormat; + protected static String rowFormat; + protected static String transactionFormat; + protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + static { + StringBuilder sb = new StringBuilder(); + sb.append(SEP) + .append("-------------Batch-------------") + .append(SEP) + .append("* Batch Id: [{}] ,count : [{}] , Mem size : [{}] , Time : {}") + .append(SEP) + .append("* Start : [{}] ") + .append(SEP) + .append("* End : [{}] ") + .append(SEP) + .append("-------------------------------") + .append(SEP); + contextFormat = sb.toString(); + + sb = new StringBuilder(); + sb.append(SEP) + .append("+++++++++++++Row+++++++++++++>>>") + .append("binlog[{}:{}] , name[{},{}] , eventType : {} , executeTime : {} , delay : {}ms") + .append(SEP); + rowFormat = sb.toString(); + + sb = new StringBuilder(); + sb.append(SEP) + .append("===========Transaction {} : {}=======>>>") + .append("binlog[{}:{}] , executeTime : {} , delay : {}ms") + .append(SEP); + transactionFormat = sb.toString(); + } + + private String zkServers;//cluster + private String address;//single，ip:port + private String destination; + private String username; + private String password; + private int batchSize = 5 * 1024; + private String filter = "";//同canal filter，用于过滤database或者table的相关数据。 + protected boolean debug = false;//开启debug，会把每条消息的详情打印 + + //1:retry，重试，重试默认为3次，由retryTimes参数决定，如果重试次数达到阈值，则跳过，并且记录日志。 + //2:ignore,直接忽略，不重试，记录日志。 + protected int exceptionStrategy = 1; + protected int retryTimes = 3; + protected int waitingTime = 100;//当binlog没有数据时，主线程等待的时间，单位ms,大于0 + + + protected CanalConnector connector; + protected Thread thread; + + protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { + + public void uncaughtException(Thread t, Throwable e) { + logger.error("process message has an error", e); + } + }; + + @Override + public void afterPropertiesSet() { + if (waitingTime <= 0) { + throw new IllegalArgumentException("waitingTime must be greater than 0"); + } + if (ExceptionStrategy.codeOf(exceptionStrategy) == null) { + throw new IllegalArgumentException("exceptionStrategy is not valid,1 or 2"); + } + start(); + } + + @Override + public void start() { + if (running) { + return; + } + super.start(); + initConnector(); + + thread = new Thread(new Runnable() { + + public void run() { + process(); + } + }); + + thread.setUncaughtExceptionHandler(handler); + thread.start(); + } + + @Override + public void stop() { + if (!running) { + return; + } + super.stop(); + quietlyStop(thread); + } + + protected void quietlyStop(Thread task) { + if (task != null) { + task.interrupt(); + try { + task.join(); + } catch (InterruptedException e) { + // ignore + } + } + } + + public void process() { + int times = 0; + while (running) { + try { + sleepWhenFailed(times); + //after block, should check the status of thread. + if (!running) { + break; + } + MDC.put("destination", destination); + connector.connect(); + connector.subscribe(filter); + connector.rollback(); + times = 0;//reset; + + while (running) { + // 获取指定数量的数据，不确认 + Message message = connector.getWithoutAck(batchSize); + + long batchId = message.getId(); + int size = message.getEntries().size(); + + if (batchId == -1 || size == 0) { + try { + Thread.sleep(waitingTime); + } catch (InterruptedException e) { + // + } + continue; + } + //logger + printBatch(message, batchId); + + processMessage(message); + + } + } catch (Exception e) { + logger.error("process error!", e); + if (times > 20) { + times = 0; + } + times++; + } finally { + connector.disconnect(); + MDC.remove("destination"); + } + } + } + + protected abstract void processMessage(Message message); + + + private void initConnector() { + if (zkServers != null && zkServers.length() > 0) { + connector = CanalConnectors.newClusterConnector(zkServers, destination, username, password); + } else if (address != null) { + String[] segments = address.split(":"); + SocketAddress socketAddress = new InetSocketAddress(segments[0], Integer.valueOf(segments[1])); + connector = CanalConnectors.newSingleConnector(socketAddress, destination, username, password); + } else { + throw new IllegalArgumentException("zkServers or address cant be null at same time,you should specify one of them!"); + } + + } + + /** + * 用于控制当连接异常时，重试的策略，我们不应该每次都是立即重试，否则将可能导致大量的错误，在空转时导致CPU过高的问题 + * sleep策略基于简单的累加 + * + * @param times + */ + private void sleepWhenFailed(int times) { + if (times <= 0) { + return; + } + try { + int sleepTime = 1000 + times * 100;//最大sleep 3s。 + Thread.sleep(sleepTime); + } catch (Exception ex) { + // + } + } + + /** + * 打印当前batch的摘要信息 + * + * @param message + * @param batchId + */ + protected void printBatch(Message message, long batchId) { + if (!debug) { + return; + } + List entries = message.getEntries(); + if (CollectionUtils.isEmpty(entries)) { + return; + } + + long memSize = 0; + for (CanalEntry.Entry entry : entries) { + memSize += entry.getHeader().getEventLength(); + } + int size = entries.size(); + String startPosition = buildPosition(entries.get(0)); + String endPosition = buildPosition(message.getEntries().get(size - 1)); + + SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT); + logger.info(contextFormat, new Object[]{batchId, size, memSize, format.format(new Date()), startPosition, endPosition}); + } + + protected String buildPosition(CanalEntry.Entry entry) { + CanalEntry.Header header = entry.getHeader(); + long time = header.getExecuteTime(); + Date date = new Date(time); + SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT); + StringBuilder sb = new StringBuilder(); + sb.append(header.getLogfileName()) + .append(":") + .append(header.getLogfileOffset()) + .append(":") + .append(header.getExecuteTime()) + .append("(") + .append(format.format(date)) + .append(")"); + return sb.toString(); + } + + /** + * default，only logging information + * + * @param entry + */ + protected void transactionBegin(CanalEntry.Entry entry) { + if (!debug) { + return; + } + try { + CanalEntry.TransactionBegin begin = CanalEntry.TransactionBegin.parseFrom(entry.getStoreValue()); + // 打印事务头信息，执行的线程id，事务耗时 + CanalEntry.Header header = entry.getHeader(); + long executeTime = header.getExecuteTime(); + long delayTime = new Date().getTime() - executeTime; + logger.info(transactionFormat, + new Object[]{"begin", begin.getTransactionId(), header.getLogfileName(), + String.valueOf(header.getLogfileOffset()), + String.valueOf(header.getExecuteTime()), String.valueOf(delayTime)}); + } catch (Exception e) { + logger.error("parse event has an error , data:" + entry.toString(), e); + } + } + + protected void transactionEnd(CanalEntry.Entry entry) { + if (!debug) { + return; + } + try { + CanalEntry.TransactionEnd end = CanalEntry.TransactionEnd.parseFrom(entry.getStoreValue()); + // 打印事务提交信息，事务id + CanalEntry.Header header = entry.getHeader(); + long executeTime = header.getExecuteTime(); + long delayTime = new Date().getTime() - executeTime; + + logger.info(transactionFormat, + new Object[]{"end", end.getTransactionId(), header.getLogfileName(), + String.valueOf(header.getLogfileOffset()), + String.valueOf(header.getExecuteTime()), String.valueOf(delayTime)}); + } catch (Exception e) { + logger.error("parse event has an error , data:" + entry.toString(), e); + } + } + + /** + * 判断事件类型为DML 数据 + * + * @param eventType + * @return + */ + protected boolean isDML(CanalEntry.EventType eventType) { + switch (eventType) { + case INSERT: + case UPDATE: + case DELETE: + return true; + default: + return false; + } + } + + /** + * 处理 DDL数据 + * + * @param header + * @param eventType + * @param sql + */ + + protected void processDDL(CanalEntry.Header header, CanalEntry.EventType eventType, String sql) { + if (!debug) { + return; + } + String table = header.getSchemaName() + "." + header.getTableName(); + //对于DDL，直接执行，因为没有行变更数据 + switch (eventType) { + case CREATE: + logger.warn("parse create table event, table: {}, sql: {}", table, sql); + return; + case ALTER: + logger.warn("parse alter table event, table: {}, sql: {}", table, sql); + return; + case TRUNCATE: + logger.warn("parse truncate table event, table: {}, sql: {}", table, sql); + return; + case ERASE: + case QUERY: + logger.warn("parse event : {}, sql: {} . ignored!", eventType.name(), sql); + return; + case RENAME: + logger.warn("parse rename table event, table: {}, sql: {}", table, sql); + return; + case CINDEX: + logger.warn("parse create index event, table: {}, sql: {}", table, sql); + return; + case DINDEX: + logger.warn("parse delete index event, table: {}, sql: {}", table, sql); + return; + default: + logger.warn("parse unknown event: {}, table: {}, sql: {}", new String[]{eventType.name(), table, sql}); + break; + } + } + + /** + * 强烈建议捕获异常，非上述已列出的其他操作，非核心 + * 除了“insert”、“update”、“delete”操作之外的，其他类型的操作. + * 默认实现为“无操作” + * + * @param header 可以从header中获得schema、table的名称 + * @param sql + */ + public void whenOthers(CanalEntry.Header header, String sql) { + String schema = header.getSchemaName(); + String table = header.getTableName(); + logger.error("ignore event,schema: {},table: {},SQL: {}", new String[]{schema, table, sql}); + } + + public enum ExceptionStrategy { + RETRY(1), IGNORE(2); + public int code; + + ExceptionStrategy(int code) { + this.code = code; + } + + public static ExceptionStrategy codeOf(Integer code) { + if (code != null) { + for (ExceptionStrategy e : ExceptionStrategy.values()) { + if (e.code == code) { + return e; + } + } + } + return null; + } + } + + public String getZkServers() { + return zkServers; + } + + public void setZkServers(String zkServers) { + this.zkServers = zkServers; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + public boolean isDebug() { + return debug; + } + + public void setDebug(boolean debug) { + this.debug = debug; + } + + public int getExceptionStrategy() { + return exceptionStrategy; + } + + public void setExceptionStrategy(int exceptionStrategy) { + this.exceptionStrategy = exceptionStrategy; + } + + public int getRetryTimes() { + return retryTimes; + } + + public void setRetryTimes(int retryTimes) { + this.retryTimes = retryTimes; + } + + public int getWaitingTime() { + return waitingTime; + } + + public void setWaitingTime(int waitingTime) { + this.waitingTime = waitingTime; + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/MysqlLoadLauncher.java b/example/src/main/java/com/alibaba/otter/canal/example/db/MysqlLoadLauncher.java new file mode 100644 index 00000000..b9936106 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/MysqlLoadLauncher.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.example.db; + +import com.alibaba.otter.canal.example.db.mysql.MysqlClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MysqlLoadLauncher { + private static final Logger logger = LoggerFactory.getLogger(MysqlLoadLauncher.class); + + public static void main(String[] args) { + try { + logger.info("## start the canal mysql client."); + final MysqlClient client = ServiceLocator.getMysqlClient(); + logger.info("## the canal consumer is running now ......"); + client.start(); + Runtime.getRuntime().addShutdownHook(new Thread() { + + public void run() { + try { + logger.info("## stop the canal consumer"); + client.stop(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping canal consumer:\n{}", e); + } finally { + logger.info("## canal consumer is down."); + } + } + + }); + } catch (Throwable e) { + logger.error("## Something goes wrong when starting up the canal consumer:\n{}", e); + System.exit(0); + } + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/ServiceLocator.java b/example/src/main/java/com/alibaba/otter/canal/example/db/ServiceLocator.java new file mode 100644 index 00000000..39ad4c62 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/ServiceLocator.java @@ -0,0 +1,44 @@ +package com.alibaba.otter.canal.example.db; + +import com.alibaba.otter.canal.example.db.mysql.MysqlClient; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.util.Assert; + +public class ServiceLocator implements DisposableBean { + + private static ApplicationContext applicationContext = null; + + static { + try { + applicationContext = new ClassPathXmlApplicationContext("classpath:client-spring.xml"); + } catch (RuntimeException e) { + throw e; + } + } + + private static T getBean(String name) { + assertContextInjected(); + return (T) applicationContext.getBean(name); + } + + + private static void clearHolder() { + ServiceLocator.applicationContext = null; + } + + @Override + public void destroy() throws Exception { + ServiceLocator.clearHolder(); + } + + private static void assertContextInjected() { + Assert.state(applicationContext != null, "ApplicationContext not set"); + } + + + public static MysqlClient getMysqlClient() { + return getBean("mysqlClient"); + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractDbDialect.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractDbDialect.java new file mode 100644 index 00000000..ec99b316 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractDbDialect.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2010-2101 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.otter.canal.example.db.dialect; + +import com.alibaba.otter.canal.example.db.utils.DdlUtils; +import com.google.common.base.Function; +import com.google.common.collect.MigrateMap; +import org.apache.commons.lang.exception.NestableRuntimeException; +import org.apache.ddlutils.model.Table; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.ConnectionCallback; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.support.lob.LobHandler; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.util.Assert; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public abstract class AbstractDbDialect implements DbDialect { + + protected int databaseMajorVersion; + protected int databaseMinorVersion; + protected String databaseName; + protected JdbcTemplate jdbcTemplate; + protected TransactionTemplate transactionTemplate; + protected LobHandler lobHandler; + protected Map, Table> tables; + + public AbstractDbDialect(final JdbcTemplate jdbcTemplate, LobHandler lobHandler) { + this.jdbcTemplate = jdbcTemplate; + this.lobHandler = lobHandler; + // 初始化transction + this.transactionTemplate = new TransactionTemplate(); + transactionTemplate.setTransactionManager(new DataSourceTransactionManager(jdbcTemplate.getDataSource())); + transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + // 初始化一些数据 + jdbcTemplate.execute(new ConnectionCallback() { + + public Object doInConnection(Connection c) throws SQLException, DataAccessException { + DatabaseMetaData meta = c.getMetaData(); + databaseName = meta.getDatabaseProductName(); + databaseMajorVersion = meta.getDatabaseMajorVersion(); + databaseMinorVersion = meta.getDatabaseMinorVersion(); + + return null; + } + }); + + initTables(jdbcTemplate); + } + + public Table findTable(String schema, String table, boolean useCache) { + List key = Arrays.asList(schema, table); + if (useCache == false) { + tables.remove(key); + } + + return tables.get(key); + } + + public Table findTable(String schema, String table) { + return findTable(schema, table, true); + } + + public LobHandler getLobHandler() { + return lobHandler; + } + + public JdbcTemplate getJdbcTemplate() { + return jdbcTemplate; + } + + public TransactionTemplate getTransactionTemplate() { + return transactionTemplate; + } + + private void initTables(final JdbcTemplate jdbcTemplate) { + this.tables = MigrateMap.makeComputingMap(new Function, Table>() { + + public Table apply(List names) { + Assert.isTrue(names.size() == 2); + try { + Table table = DdlUtils.findTable(jdbcTemplate, names.get(0), names.get(0), names.get(1)); + if (table == null) { + throw new NestableRuntimeException("no found table [" + names.get(0) + "." + names.get(1) + + "] , pls check"); + } else { + return table; + } + } catch (Exception e) { + throw new NestableRuntimeException("find table [" + names.get(0) + "." + names.get(1) + "] error", + e); + } + } + }); + } + + +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractSqlTemplate.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractSqlTemplate.java new file mode 100644 index 00000000..fbbecabe --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/AbstractSqlTemplate.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2010-2101 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.otter.canal.example.db.dialect; + +/** + * 默认的基于标准SQL实现的CRUD sql封装 + * + * @author jianghang 2011-10-27 下午01:37:00 + * @version 4.0.0 + */ +public abstract class AbstractSqlTemplate implements SqlTemplate { + + private static final String DOT = "."; + + public String getSelectSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { + StringBuilder sql = new StringBuilder("select "); + int size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(columnNames[i])).append((i + 1 < size) ? " , " : ""); + } + + sql.append(" from ").append(getFullName(schemaName, tableName)).append(" where ( "); + appendColumnEquals(sql, pkNames, "and"); + sql.append(" ) "); + return sql.toString().intern();// 不使用intern，避免方法区内存消耗过多 + } + + public String getUpdateSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { + StringBuilder sql = new StringBuilder("update " + getFullName(schemaName, tableName) + " set "); + appendColumnEquals(sql, columnNames, ","); + sql.append(" where ("); + appendColumnEquals(sql, pkNames, "and"); + sql.append(")"); + return sql.toString().intern(); // 不使用intern，避免方法区内存消耗过多 + } + + public String getInsertSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { + StringBuilder sql = new StringBuilder("insert into " + getFullName(schemaName, tableName) + "("); + String[] allColumns = new String[pkNames.length + columnNames.length]; + System.arraycopy(columnNames, 0, allColumns, 0, columnNames.length); + System.arraycopy(pkNames, 0, allColumns, columnNames.length, pkNames.length); + + int size = allColumns.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(allColumns[i])).append((i + 1 < size) ? "," : ""); + } + + sql.append(") values ("); + appendColumnQuestions(sql, allColumns); + sql.append(")"); + return sql.toString().intern();// intern优化，避免出现大量相同的字符串 + } + + public String getDeleteSql(String schemaName, String tableName, String[] pkNames) { + StringBuilder sql = new StringBuilder("delete from " + getFullName(schemaName, tableName) + " where "); + appendColumnEquals(sql, pkNames, "and"); + return sql.toString().intern();// intern优化，避免出现大量相同的字符串 + } + + protected String getFullName(String schemaName, String tableName) { + StringBuilder sb = new StringBuilder(); + if (schemaName != null) { + sb.append(appendEscape(schemaName)).append(DOT); + } + sb.append(appendEscape(tableName)); + return sb.toString().intern(); + } + + // ================ helper method ============ + + protected String appendEscape(String columnName) { + return columnName; + } + + protected void appendColumnQuestions(StringBuilder sql, String[] columns) { + int size = columns.length; + for (int i = 0; i < size; i++) { + sql.append("?").append((i + 1 < size) ? " , " : ""); + } + } + + protected void appendColumnEquals(StringBuilder sql, String[] columns, String separator) { + int size = columns.length; + for (int i = 0; i < size; i++) { + sql.append(" ").append(appendEscape(columns[i])).append(" = ").append("? "); + if (i != size - 1) { + sql.append(separator); + } + } + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/DbDialect.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/DbDialect.java new file mode 100644 index 00000000..060f476d --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/DbDialect.java @@ -0,0 +1,20 @@ +package com.alibaba.otter.canal.example.db.dialect; + +import org.apache.ddlutils.model.Table; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.lob.LobHandler; +import org.springframework.transaction.support.TransactionTemplate; + +public interface DbDialect { + + LobHandler getLobHandler(); + + JdbcTemplate getJdbcTemplate(); + + TransactionTemplate getTransactionTemplate(); + + Table findTable(String schema, String table); + + Table findTable(String schema, String table, boolean useCache); + +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/SqlTemplate.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/SqlTemplate.java new file mode 100644 index 00000000..a90bd5ce --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/SqlTemplate.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2010-2101 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.otter.canal.example.db.dialect; + +/** + * sql构造模板操作 + * + * @author jianghang 2011-10-27 下午01:31:15 + * @version 4.0.0 + */ +public interface SqlTemplate { + + public String getSelectSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); + + public String getUpdateSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); + + public String getDeleteSql(String schemaName, String tableName, String[] pkNames); + + public String getInsertSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); + + /** + * 获取对应的mergeSql + */ + public String getMergeSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, + String[] viewColumnNames, boolean updatePks); +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/TableType.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/TableType.java new file mode 100644 index 00000000..58cf662a --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/TableType.java @@ -0,0 +1,93 @@ +package com.alibaba.otter.canal.example.db.dialect; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * An enumeration wrapper around JDBC table types. + */ +public enum TableType { + + /** + * Unknown + */ + unknown, + + /** + * System table + */ + system_table, + + /** + * Global temporary + */ + global_temporary, + + /** + * Local temporary + */ + local_temporary, + + /** + * Table + */ + table, + + /** + * View + */ + view, + + /** + * Alias + */ + alias, + + /** + * Synonym + */ + synonym,; + + /** + * Converts an array of table types to an array of their corresponding string values. + * + * @param tableTypes Array of table types + * @return Array of string table types + */ + public static String[] toStrings(final TableType[] tableTypes) { + if ((tableTypes == null) || (tableTypes.length == 0)) { + return new String[0]; + } + + final List tableTypeStrings = new ArrayList(tableTypes.length); + + for (final TableType tableType : tableTypes) { + if (tableType != null) { + tableTypeStrings.add(tableType.toString().toUpperCase(Locale.ENGLISH)); + } + } + + return tableTypeStrings.toArray(new String[tableTypeStrings.size()]); + } + + /** + * Converts an array of string table types to an array of their corresponding enumeration values. + * + * @param tableTypeStrings Array of string table types + * @return Array of table types + */ + public static TableType[] valueOf(final String[] tableTypeStrings) { + if ((tableTypeStrings == null) || (tableTypeStrings.length == 0)) { + return new TableType[0]; + } + + final List tableTypes = new ArrayList(tableTypeStrings.length); + + for (final String tableTypeString : tableTypeStrings) { + tableTypes.add(valueOf(tableTypeString.toLowerCase(Locale.ENGLISH))); + } + + return tableTypes.toArray(new TableType[tableTypes.size()]); + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlDialect.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlDialect.java new file mode 100644 index 00000000..7bdfe1df --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlDialect.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2010-2101 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.otter.canal.example.db.dialect.mysql; + +import com.alibaba.otter.canal.example.db.dialect.AbstractDbDialect; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.lob.LobHandler; + +public class MysqlDialect extends AbstractDbDialect { + + public MysqlDialect(JdbcTemplate jdbcTemplate, LobHandler lobHandler) { + super(jdbcTemplate, lobHandler); + } + + public boolean isEmptyStringNulled() { + return false; + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlSqlTemplate.java b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlSqlTemplate.java new file mode 100644 index 00000000..0cf987ff --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlSqlTemplate.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2010-2101 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.otter.canal.example.db.dialect.mysql; + +import com.alibaba.otter.canal.example.db.dialect.AbstractSqlTemplate; + +/** + * mysql sql生成模板 + * + * @author jianghang 2011-10-27 下午01:41:20 + * @version 4.0.0 + */ +public class MysqlSqlTemplate extends AbstractSqlTemplate { + + private static final String ESCAPE = "`"; + + public String getMergeSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, + String[] viewColumnNames, boolean includePks) { + StringBuilder sql = new StringBuilder("insert into " + getFullName(schemaName, tableName) + "("); + int size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(columnNames[i])).append(" , "); + } + size = pkNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(pkNames[i])).append((i + 1 < size) ? " , " : ""); + } + + sql.append(") values ("); + size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append("?").append(" , "); + } + size = pkNames.length; + for (int i = 0; i < size; i++) { + sql.append("?").append((i + 1 < size) ? " , " : ""); + } + sql.append(")"); + sql.append(" on duplicate key update "); + + size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(columnNames[i])) + .append("=values(") + .append(appendEscape(columnNames[i])) + .append(")"); + if (includePks) { + sql.append(" , "); + } else { + sql.append((i + 1 < size) ? " , " : ""); + } + } + + if (includePks) { + // mysql merge sql匹配了uniqe / primary key时都会执行update，所以需要更新pk信息 + size = pkNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(pkNames[i])).append("=values(").append(appendEscape(pkNames[i])).append(")"); + sql.append((i + 1 < size) ? " , " : ""); + } + } + + return sql.toString().intern();// intern优化，避免出现大量相同的字符串 + } + + protected String appendEscape(String columnName) { + return ESCAPE + columnName + ESCAPE; + } + +} \ No newline at end of file diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/mysql/AbstractMysqlClient.java b/example/src/main/java/com/alibaba/otter/canal/example/db/mysql/AbstractMysqlClient.java new file mode 100644 index 00000000..608b81e6 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/mysql/AbstractMysqlClient.java @@ -0,0 +1,207 @@ +package com.alibaba.otter.canal.example.db.mysql; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.example.db.AbstractDbClient; +import com.alibaba.otter.canal.example.db.dialect.DbDialect; +import com.alibaba.otter.canal.example.db.dialect.mysql.MysqlDialect; +import com.alibaba.otter.canal.example.db.dialect.mysql.MysqlSqlTemplate; +import com.alibaba.otter.canal.example.db.dialect.SqlTemplate; +import com.alibaba.otter.canal.example.db.utils.SqlUtils; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import org.apache.commons.lang.StringUtils; +import org.apache.ddlutils.model.Column; +import org.apache.ddlutils.model.Table; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementSetter; +import org.springframework.jdbc.core.StatementCreatorUtils; +import org.springframework.jdbc.support.lob.DefaultLobHandler; +import org.springframework.jdbc.support.lob.LobCreator; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; + +import javax.sql.DataSource; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public abstract class AbstractMysqlClient extends AbstractDbClient { + + private DataSource dataSource; + + private DbDialect dbDialect; + private SqlTemplate sqlTemplate; + + protected Integer execute(final CanalEntry.Header header, final List columns) { + final String sql = getSql(header, columns); + final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator(); + dbDialect.getTransactionTemplate().execute(new TransactionCallback() { + + public Object doInTransaction(TransactionStatus status) { + try { + JdbcTemplate template = dbDialect.getJdbcTemplate(); + int affect = template.update(sql, new PreparedStatementSetter() { + + public void setValues(PreparedStatement ps) throws SQLException { + doPreparedStatement(ps, dbDialect, lobCreator, header, columns); + } + }); + return affect; + } finally { + lobCreator.close(); + } + } + }); + return 0; + } + + private String getSql(CanalEntry.Header header, List columns) { + List pkNames = new ArrayList<>(); + List colNames = new ArrayList<>(); + for (CanalEntry.Column column : columns) { + if (column.getIsKey()) { + pkNames.add(column.getName()); + } else { + colNames.add(column.getName()); + } + } + String sql = ""; + CanalEntry.EventType eventType = header.getEventType(); + switch (eventType) { + case INSERT: + sql = sqlTemplate.getInsertSql(header.getSchemaName(), header.getTableName(), pkNames.toArray(new String[]{}), colNames.toArray(new String[]{})); + break; + case UPDATE: + sql = sqlTemplate.getUpdateSql(header.getSchemaName(), header.getTableName(), pkNames.toArray(new String[]{}), colNames.toArray(new String[]{})); + break; + case DELETE: + sql = sqlTemplate.getDeleteSql(header.getSchemaName(), header.getTableName(), pkNames.toArray(new String[]{})); + } + logger.info("Execute sql: {}", sql); + return sql; + } + + private void doPreparedStatement(PreparedStatement ps, DbDialect dbDialect, LobCreator lobCreator, + CanalEntry.Header header, List columns) throws SQLException { + + List rebuildColumns = new ArrayList<>(columns.size()); + + List keyColumns = new ArrayList<>(columns.size()); + List notKeyColumns = new ArrayList<>(columns.size()); + for (CanalEntry.Column column : columns) { + if (column.getIsKey()) { + keyColumns.add(column); + } else { + notKeyColumns.add(column); + } + } + CanalEntry.EventType eventType = header.getEventType(); + switch (eventType) { + case INSERT: + case UPDATE: + // insert/update语句对应的字段数序都是将主键排在后面 + rebuildColumns.addAll(notKeyColumns); + rebuildColumns.addAll(keyColumns); + break; + case DELETE: + rebuildColumns.addAll(keyColumns); + } + + // 获取一下当前字段名的数据是否必填 + Table table = dbDialect.findTable(header.getSchemaName(), header.getTableName()); + Map isRequiredMap = new HashMap(); + for (Column tableColumn : table.getColumns()) { + isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired()); + } + + List values = new ArrayList<>(rebuildColumns.size()); + for (int i = 0; i < rebuildColumns.size(); i++) { + int paramIndex = i + 1; + CanalEntry.Column column = rebuildColumns.get(i); + int sqlType = column.getSqlType(); + + Boolean isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getName())); + if (isRequired == null) { + // 清理一下目标库的表结构,二次检查一下 + table = dbDialect.findTable(header.getSchemaName(), header.getTableName()); + + isRequiredMap = new HashMap<>(); + for (Column tableColumn : table.getColumns()) { + isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired()); + } + + isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getName())); + if (isRequired == null) { + throw new CanalClientException(String.format("column name %s is not found in Table[%s]", + column.getName(), + table.toString())); + } + } + + Object param; + if (sqlType == Types.TIME || sqlType == Types.TIMESTAMP || sqlType == Types.DATE) { + // 解决mysql的0000-00-00 00:00:00问题，直接依赖mysql + // driver进行处理，如果转化为Timestamp会出错 + param = column.getValue(); + if (param instanceof String && StringUtils.isEmpty(String.valueOf(param))) { + param = null; + } + } else { + param = SqlUtils.stringToSqlValue(column.getValue(), + sqlType, + isRequired, + column.getIsNull()); + } + + try { + switch (sqlType) { + case Types.CLOB: + lobCreator.setClobAsString(ps, paramIndex, (String) param); + break; + case Types.BLOB: + lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param); + break; + case Types.TIME: + case Types.TIMESTAMP: + case Types.DATE: + ps.setObject(paramIndex, param); + break; + case Types.BIT: + StatementCreatorUtils.setParameterValue(ps, paramIndex, Types.DECIMAL, null, param); + break; + default: + StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); + break; + } + values.add(param); + } catch (SQLException ex) { + logger.error("## SetParam error , [sqltype={}, value={}]", + new Object[]{sqlType, param}); + throw ex; + } + } + logger.info("## sql values: {}", JSON.toJSONString(values)); + } + + @Override + public void afterPropertiesSet() { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + DefaultLobHandler lobHandler = new DefaultLobHandler(); + lobHandler.setStreamAsLob(true); + dbDialect = new MysqlDialect(jdbcTemplate, lobHandler); + sqlTemplate = new MysqlSqlTemplate(); + } + + public DataSource getDataSource() { + return dataSource; + } + + public void setDataSource(DataSource dataSource) { + this.dataSource = dataSource; + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/mysql/MysqlClient.java b/example/src/main/java/com/alibaba/otter/canal/example/db/mysql/MysqlClient.java new file mode 100644 index 00000000..f16b6146 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/mysql/MysqlClient.java @@ -0,0 +1,23 @@ +package com.alibaba.otter.canal.example.db.mysql; + +import com.alibaba.otter.canal.protocol.CanalEntry; + +import java.util.List; + +public class MysqlClient extends AbstractMysqlClient { + + @Override + public void insert(CanalEntry.Header header, List afterColumns) { + execute(header, afterColumns); + } + + @Override + public void update(CanalEntry.Header header, List afterColumns) { + execute(header, afterColumns); + } + + @Override + public void delete(CanalEntry.Header header, List beforeColumns) { + execute(header, beforeColumns); + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/utils/ByteArrayConverter.java b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/ByteArrayConverter.java new file mode 100644 index 00000000..6bd33686 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/ByteArrayConverter.java @@ -0,0 +1,50 @@ +package com.alibaba.otter.canal.example.db.utils; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.Converter; +import org.apache.commons.beanutils.converters.ArrayConverter; +import org.apache.commons.beanutils.converters.ByteConverter; + +public class ByteArrayConverter implements Converter { + + public static final Converter SQL_BYTES = new ByteArrayConverter(null); + private static final Converter converter = new ArrayConverter(byte[].class, new ByteConverter()); + + protected final Object defaultValue; + protected final boolean useDefault; + + public ByteArrayConverter() { + this.defaultValue = null; + this.useDefault = false; + } + + public ByteArrayConverter(Object defaultValue) { + this.defaultValue = defaultValue; + this.useDefault = true; + } + + public Object convert(Class type, Object value) { + if (value == null) { + if (useDefault) { + return (defaultValue); + } else { + throw new ConversionException("No value specified"); + } + } + + if (value instanceof byte[]) { + return (value); + } + + // BLOB类型，canal直接存储为String("ISO-8859-1") + if (value instanceof String) { + try { + return ((String) value).getBytes("ISO-8859-1"); + } catch (Exception e) { + throw new ConversionException(e); + } + } + + return converter.convert(type, value); // byteConvertor进行转化 + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/utils/DdlUtils.java b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/DdlUtils.java new file mode 100644 index 00000000..7964b18d --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/DdlUtils.java @@ -0,0 +1,326 @@ +package com.alibaba.otter.canal.example.db.utils; + +import com.alibaba.otter.canal.example.db.dialect.TableType; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.ddlutils.model.Column; +import org.apache.ddlutils.model.Table; +import org.apache.ddlutils.platform.DatabaseMetaDataWrapper; +import org.apache.ddlutils.platform.MetaDataColumnDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.ConnectionCallback; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.JdbcUtils; + +import java.sql.*; +import java.util.*; + + +public class DdlUtils { + + private static final Logger logger = LoggerFactory.getLogger(DdlUtils.class); + private static TableType[] SUPPORTED_TABLE_TYPES = new TableType[]{TableType.view, TableType.table}; + private final static Map _defaultSizes = new HashMap(); + + static { + _defaultSizes.put(new Integer(1), "254"); + _defaultSizes.put(new Integer(12), "254"); + _defaultSizes.put(new Integer(-1), "254"); + _defaultSizes.put(new Integer(-2), "254"); + _defaultSizes.put(new Integer(-3), "254"); + _defaultSizes.put(new Integer(-4), "254"); + _defaultSizes.put(new Integer(4), "32"); + _defaultSizes.put(new Integer(-5), "64"); + _defaultSizes.put(new Integer(7), "7,0"); + _defaultSizes.put(new Integer(6), "15,0"); + _defaultSizes.put(new Integer(8), "15,0"); + _defaultSizes.put(new Integer(3), "15,15"); + _defaultSizes.put(new Integer(2), "15,15"); + } + + + public static Table findTable(final JdbcTemplate jdbcTemplate, final String catalogName, final String schemaName, + final String tableName) { + return (Table) jdbcTemplate.execute(new ConnectionCallback() { + + public Object doInConnection(Connection con) throws SQLException, DataAccessException { + Table table = null; + DatabaseMetaDataWrapper metaData = new DatabaseMetaDataWrapper(); + try { + + DatabaseMetaData databaseMetaData = con.getMetaData(); + + metaData.setMetaData(databaseMetaData); + metaData.setTableTypes(TableType.toStrings(SUPPORTED_TABLE_TYPES)); + metaData.setCatalog(catalogName); + metaData.setSchemaPattern(schemaName); + + String convertTableName = tableName; + if (databaseMetaData.storesUpperCaseIdentifiers()) { + metaData.setCatalog(catalogName.toUpperCase()); + metaData.setSchemaPattern(schemaName.toUpperCase()); + convertTableName = tableName.toUpperCase(); + } + if (databaseMetaData.storesLowerCaseIdentifiers()) { + metaData.setCatalog(catalogName.toLowerCase()); + metaData.setSchemaPattern(schemaName.toLowerCase()); + convertTableName = tableName.toLowerCase(); + } + + ResultSet tableData = null; + try { + tableData = metaData.getTables(convertTableName); + + while ((tableData != null) && tableData.next()) { + Map values = readColumns(tableData, initColumnsForTable()); + + table = readTable(metaData, values); + if (table.getName().equalsIgnoreCase(tableName)) { + break; + } + } + } finally { + JdbcUtils.closeResultSet(tableData); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + makeAllColumnsPrimaryKeysIfNoPrimaryKeysFound(table); + return table; + } + }); + } + + /** + * Treat tables with no primary keys as a table with all primary keys. + */ + private static void makeAllColumnsPrimaryKeysIfNoPrimaryKeysFound(Table table) { + if ((table != null) && (table.getPrimaryKeyColumns() != null) && (table.getPrimaryKeyColumns().length == 0)) { + Column[] allCoumns = table.getColumns(); + + for (Column column : allCoumns) { + column.setPrimaryKey(true); + } + } + } + + private static Table readTable(DatabaseMetaDataWrapper metaData, Map values) throws SQLException { + String tableName = (String) values.get("TABLE_NAME"); + Table table = null; + + if ((tableName != null) && (tableName.length() > 0)) { + table = new Table(); + table.setName(tableName); + table.setType((String) values.get("TABLE_TYPE")); + table.setCatalog((String) values.get("TABLE_CAT")); + table.setSchema((String) values.get("TABLE_SCHEM")); + table.setDescription((String) values.get("REMARKS")); + table.addColumns(readColumns(metaData, tableName)); + + Collection primaryKeys = readPrimaryKeyNames(metaData, tableName); + + for (Object key : primaryKeys) { + Column col = table.findColumn((String) key, true); + + if (col != null) { + col.setPrimaryKey(true); + } else { + throw new NullPointerException(String.format("%s pk %s is null - %s %s", + tableName, + key, + ToStringBuilder.reflectionToString(metaData, ToStringStyle.SIMPLE_STYLE), + ToStringBuilder.reflectionToString(values, ToStringStyle.SIMPLE_STYLE))); + } + } + } + + return table; + } + + private static List initColumnsForTable() { + List result = new ArrayList(); + + result.add(new MetaDataColumnDescriptor("TABLE_NAME", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("TABLE_TYPE", Types.VARCHAR, "UNKNOWN")); + result.add(new MetaDataColumnDescriptor("TABLE_CAT", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("TABLE_SCHEM", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("REMARKS", Types.VARCHAR)); + + return result; + } + + private static List initColumnsForColumn() { + List result = new ArrayList(); + + // As suggested by Alexandre Borgoltz, we're reading the COLUMN_DEF + // first because Oracle + // has problems otherwise (it seemingly requires a LONG column to be the + // first to be read) + // See also DDLUTILS-29 + result.add(new MetaDataColumnDescriptor("COLUMN_DEF", Types.VARCHAR)); + + // we're also reading the table name so that a model reader impl can + // filter manually + result.add(new MetaDataColumnDescriptor("TABLE_NAME", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("COLUMN_NAME", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("TYPE_NAME", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("DATA_TYPE", Types.INTEGER, new Integer(Types.OTHER))); + result.add(new MetaDataColumnDescriptor("NUM_PREC_RADIX", Types.INTEGER, new Integer(10))); + result.add(new MetaDataColumnDescriptor("DECIMAL_DIGITS", Types.INTEGER, new Integer(0))); + result.add(new MetaDataColumnDescriptor("COLUMN_SIZE", Types.VARCHAR)); + result.add(new MetaDataColumnDescriptor("IS_NULLABLE", Types.VARCHAR, "YES")); + result.add(new MetaDataColumnDescriptor("REMARKS", Types.VARCHAR)); + + return result; + } + + private static List initColumnsForPK() { + List result = new ArrayList(); + + result.add(new MetaDataColumnDescriptor("COLUMN_NAME", Types.VARCHAR)); + + // we're also reading the table name so that a model reader impl can + // filter manually + result.add(new MetaDataColumnDescriptor("TABLE_NAME", Types.VARCHAR)); + + // the name of the primary key is currently only interesting to the pk + // index name resolution + result.add(new MetaDataColumnDescriptor("PK_NAME", Types.VARCHAR)); + + return result; + } + + private static List readColumns(DatabaseMetaDataWrapper metaData, String tableName) throws SQLException { + ResultSet columnData = null; + + try { + columnData = metaData.getColumns(tableName, null); + + List columns = new ArrayList(); + Map values; + + for (; columnData.next(); columns.add(readColumn(metaData, values))) { + Map tmp = readColumns(columnData, initColumnsForColumn()); + if (tableName.equalsIgnoreCase((String) tmp.get("TABLE_NAME"))) { + values = tmp; + } else { + break; + } + } + + return columns; + } finally { + JdbcUtils.closeResultSet(columnData); + } + } + + private static Column readColumn(DatabaseMetaDataWrapper metaData, Map values) throws SQLException { + Column column = new Column(); + + column.setName((String) values.get("COLUMN_NAME")); + column.setDefaultValue((String) values.get("COLUMN_DEF")); + column.setTypeCode(((Integer) values.get("DATA_TYPE")).intValue()); + + String typeName = (String) values.get("TYPE_NAME"); + // column.setType(typeName); + + if ((typeName != null) && typeName.startsWith("TIMESTAMP")) { + column.setTypeCode(Types.TIMESTAMP); + } + // modify 2013-09-25，处理下unsigned + if ((typeName != null) && StringUtils.containsIgnoreCase(typeName, "UNSIGNED")) { + // 如果为unsigned，往上调大一个量级，避免数据溢出 + switch (column.getTypeCode()) { + case Types.TINYINT: + column.setTypeCode(Types.SMALLINT); + break; + case Types.SMALLINT: + column.setTypeCode(Types.INTEGER); + break; + case Types.INTEGER: + column.setTypeCode(Types.BIGINT); + break; + case Types.BIGINT: + column.setTypeCode(Types.DECIMAL); + break; + default: + break; + } + } + + Integer precision = (Integer) values.get("NUM_PREC_RADIX"); + + if (precision != null) { + column.setPrecisionRadix(precision.intValue()); + } + + String size = (String) values.get("COLUMN_SIZE"); + + if (size == null) { + size = (String) _defaultSizes.get(new Integer(column.getTypeCode())); + } + + // we're setting the size after the precision and radix in case + // the database prefers to return them in the size value + column.setSize(size); + + int scale = 0; + Object dec_digits = values.get("DECIMAL_DIGITS"); + + if (dec_digits instanceof String) { + scale = (dec_digits == null) ? 0 : NumberUtils.toInt(dec_digits.toString()); + } else if (dec_digits instanceof Integer) { + scale = (dec_digits == null) ? 0 : (Integer) dec_digits; + } + + if (scale != 0) { + column.setScale(scale); + } + + column.setRequired("NO".equalsIgnoreCase(((String) values.get("IS_NULLABLE")).trim())); + column.setDescription((String) values.get("REMARKS")); + return column; + } + + private static Map readColumns(ResultSet resultSet, List columnDescriptors) + throws SQLException { + Map values = new HashMap(); + MetaDataColumnDescriptor descriptor; + + for (Iterator it = columnDescriptors.iterator(); it.hasNext(); values.put(descriptor.getName(), + descriptor.readColumn(resultSet))) { + descriptor = (MetaDataColumnDescriptor) it.next(); + } + + return values; + } + + private static Collection readPrimaryKeyNames(DatabaseMetaDataWrapper metaData, String tableName) + throws SQLException { + ResultSet pkData = null; + + try { + List pks = new ArrayList(); + Map values; + + for (pkData = metaData.getPrimaryKeys(tableName); pkData.next(); pks.add(readPrimaryKeyName(metaData, + values))) { + values = readColumns(pkData, initColumnsForPK()); + } + + return pks; + } finally { + JdbcUtils.closeResultSet(pkData); + } + } + + private static String readPrimaryKeyName(DatabaseMetaDataWrapper metaData, Map values) + throws SQLException { + return (String) values.get("COLUMN_NAME"); + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlTimestampConverter.java b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlTimestampConverter.java new file mode 100644 index 00000000..5743106d --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlTimestampConverter.java @@ -0,0 +1,140 @@ +package com.alibaba.otter.canal.example.db.utils; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.Converter; +import org.apache.commons.lang.time.DateFormatUtils; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; + +public class SqlTimestampConverter implements Converter { + + /** + * Field description + */ + public static final String[] DATE_FORMATS = new String[]{"yyyy-MM-dd", "HH:mm:ss", "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd hh:mm:ss.fffffffff", "EEE MMM dd HH:mm:ss zzz yyyy", + DateFormatUtils.ISO_DATETIME_FORMAT.getPattern(), + DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT.getPattern(), + DateFormatUtils.SMTP_DATETIME_FORMAT.getPattern(),}; + + public static final Converter SQL_TIMESTAMP = new SqlTimestampConverter(null); + + /** + * The default value specified to our Constructor, if any. + */ + private final Object defaultValue; + + /** + * Should we return the default value on conversion errors? + */ + private final boolean useDefault; + + /** + * Create a {@link Converter} that will throw a {@link ConversionException} if a conversion error occurs. + */ + public SqlTimestampConverter() { + this.defaultValue = null; + this.useDefault = false; + } + + /** + * Create a {@link Converter} that will return the specified default value if a conversion error occurs. + * + * @param defaultValue The default value to be returned + */ + public SqlTimestampConverter(Object defaultValue) { + this.defaultValue = defaultValue; + this.useDefault = true; + } + + /** + * Convert the specified input object into an output object of the specified type. + * + * @param type Data type to which this value should be converted + * @param value The input value to be converted + * @throws ConversionException if conversion cannot be performed successfully + */ + public Object convert(Class type, Object value) { + if (value == null) { + if (useDefault) { + return (defaultValue); + } else { + throw new ConversionException("No value specified"); + } + } + + if (value instanceof java.sql.Date && java.sql.Date.class.equals(type)) { + return value; + } else if (value instanceof java.sql.Time && java.sql.Time.class.equals(type)) { + return value; + } else if (value instanceof Timestamp && Timestamp.class.equals(type)) { + return value; + } else { + try { + if (java.sql.Date.class.equals(type)) { + return new java.sql.Date(convertTimestamp2TimeMillis(value.toString())); + } else if (java.sql.Time.class.equals(type)) { + return new java.sql.Time(convertTimestamp2TimeMillis(value.toString())); + } else if (Timestamp.class.equals(type)) { + return new Timestamp(convertTimestamp2TimeMillis(value.toString())); + } else { + return new Timestamp(convertTimestamp2TimeMillis(value.toString())); + } + } catch (Exception e) { + throw new ConversionException("Value format invalid: " + e.getMessage(), e); + } + } + + } + + private Long convertTimestamp2TimeMillis(String input) { + if (input == null) { + return null; + } + + try { + // 先处理Timestamp类型 + return Timestamp.valueOf(input).getTime(); + } catch (Exception nfe) { + try { + try { + return parseDate(input, DATE_FORMATS, Locale.ENGLISH).getTime(); + } catch (Exception err) { + return parseDate(input, DATE_FORMATS, Locale.getDefault()).getTime(); + } + } catch (Exception err) { + // 最后处理long time的情况 + return Long.parseLong(input); + } + } + } + + private Date parseDate(String str, String[] parsePatterns, Locale locale) throws ParseException { + if ((str == null) || (parsePatterns == null)) { + throw new IllegalArgumentException("Date and Patterns must not be null"); + } + + SimpleDateFormat parser = null; + ParsePosition pos = new ParsePosition(0); + + for (int i = 0; i < parsePatterns.length; i++) { + if (i == 0) { + parser = new SimpleDateFormat(parsePatterns[0], locale); + } else { + parser.applyPattern(parsePatterns[i]); + } + pos.setIndex(0); + Date date = parser.parse(str, pos); + if ((date != null) && (pos.getIndex() == str.length())) { + return date; + } + } + + throw new ParseException("Unable to parse the date: " + str, -1); + } +} diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlUtils.java b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlUtils.java new file mode 100644 index 00000000..565d5730 --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/utils/SqlUtils.java @@ -0,0 +1,315 @@ +package com.alibaba.otter.canal.example.db.utils; + +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.apache.commons.lang.StringUtils; + +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.*; +import java.util.HashMap; +import java.util.Map; + +public class SqlUtils { + + public static final String REQUIRED_FIELD_NULL_SUBSTITUTE = " "; + public static final String SQLDATE_FORMAT = "yyyy-MM-dd"; + public static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss"; + private static final Map> sqlTypeToJavaTypeMap = new HashMap>(); + private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); + + static { + // regist Converter + convertUtilsBean.register(SqlTimestampConverter.SQL_TIMESTAMP, Date.class); + convertUtilsBean.register(SqlTimestampConverter.SQL_TIMESTAMP, Time.class); + convertUtilsBean.register(SqlTimestampConverter.SQL_TIMESTAMP, Timestamp.class); + convertUtilsBean.register(ByteArrayConverter.SQL_BYTES, byte[].class); + + // bool + sqlTypeToJavaTypeMap.put(Types.BOOLEAN, Boolean.class); + + // int + sqlTypeToJavaTypeMap.put(Types.TINYINT, Integer.class); + sqlTypeToJavaTypeMap.put(Types.SMALLINT, Integer.class); + sqlTypeToJavaTypeMap.put(Types.INTEGER, Integer.class); + + // long + sqlTypeToJavaTypeMap.put(Types.BIGINT, Long.class); + // mysql bit最多64位，无符号 + sqlTypeToJavaTypeMap.put(Types.BIT, BigInteger.class); + + // decimal + sqlTypeToJavaTypeMap.put(Types.REAL, Float.class); + sqlTypeToJavaTypeMap.put(Types.FLOAT, Float.class); + sqlTypeToJavaTypeMap.put(Types.DOUBLE, Double.class); + sqlTypeToJavaTypeMap.put(Types.NUMERIC, BigDecimal.class); + sqlTypeToJavaTypeMap.put(Types.DECIMAL, BigDecimal.class); + + // date + sqlTypeToJavaTypeMap.put(Types.DATE, Date.class); + sqlTypeToJavaTypeMap.put(Types.TIME, Time.class); + sqlTypeToJavaTypeMap.put(Types.TIMESTAMP, Timestamp.class); + + // blob + sqlTypeToJavaTypeMap.put(Types.BLOB, byte[].class); + + // byte[] + sqlTypeToJavaTypeMap.put(Types.REF, byte[].class); + sqlTypeToJavaTypeMap.put(Types.OTHER, byte[].class); + sqlTypeToJavaTypeMap.put(Types.ARRAY, byte[].class); + sqlTypeToJavaTypeMap.put(Types.STRUCT, byte[].class); + sqlTypeToJavaTypeMap.put(Types.SQLXML, byte[].class); + sqlTypeToJavaTypeMap.put(Types.BINARY, byte[].class); + sqlTypeToJavaTypeMap.put(Types.DATALINK, byte[].class); + sqlTypeToJavaTypeMap.put(Types.DISTINCT, byte[].class); + sqlTypeToJavaTypeMap.put(Types.VARBINARY, byte[].class); + sqlTypeToJavaTypeMap.put(Types.JAVA_OBJECT, byte[].class); + sqlTypeToJavaTypeMap.put(Types.LONGVARBINARY, byte[].class); + + // String + sqlTypeToJavaTypeMap.put(Types.CHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.VARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.LONGVARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.LONGNVARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.NCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.NVARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.NCLOB, String.class); + sqlTypeToJavaTypeMap.put(Types.CLOB, String.class); + } + + /** + * 将指定java.sql.Types的ResultSet value转换成相应的String + * + * @param rs + * @param index + * @param sqlType + * @return + * @throws SQLException + */ + public static String sqlValueToString(ResultSet rs, int index, int sqlType) throws SQLException { + Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); + if (requiredType == null) { + throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType); + } + + return getResultSetValue(rs, index, requiredType); + } + + /** + * sqlValueToString方法的逆向过程 + * + * @param value + * @param sqlType + * @param isTextRequired + * @param isEmptyStringNulled + * @return + */ + public static Object stringToSqlValue(String value, int sqlType, boolean isRequired, boolean isEmptyStringNulled) { + // 设置变量 + String sourceValue = value; + if (SqlUtils.isTextType(sqlType)) { + if ((sourceValue == null) || (StringUtils.isEmpty(sourceValue) && isEmptyStringNulled)) { + return isRequired ? REQUIRED_FIELD_NULL_SUBSTITUTE : null; + } else { + return sourceValue; + } + } else { + if (StringUtils.isEmpty(sourceValue)) { + return null; + } else { + Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); + if (requiredType == null) { + throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType); + } else if (requiredType.equals(String.class)) { + return sourceValue; + } else if (isNumeric(sqlType)) { + return convertUtilsBean.convert(sourceValue.trim(), requiredType); + } else { + return convertUtilsBean.convert(sourceValue, requiredType); + } + } + } + } + + public static String encoding(String source, int sqlType, String sourceEncoding, String targetEncoding) { + switch (sqlType) { + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + case Types.CLOB: + case Types.NCLOB: + if (false == StringUtils.isEmpty(source)) { + String fromEncoding = StringUtils.isBlank(sourceEncoding) ? "UTF-8" : sourceEncoding; + String toEncoding = StringUtils.isBlank(targetEncoding) ? "UTF-8" : targetEncoding; + + // if (false == StringUtils.equalsIgnoreCase(fromEncoding, + // toEncoding)) { + try { + return new String(source.getBytes(fromEncoding), toEncoding); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + // } + } + } + + return source; + } + + /** + * Retrieve a JDBC column value from a ResultSet, using the specified value + * type. + *

    + * Uses the specifically typed ResultSet accessor methods, falling back to + * {@link #getResultSetValue(ResultSet, int)} for unknown types. + *

    + * Note that the returned value may not be assignable to the specified + * required type, in case of an unknown type. Calling code needs to deal + * with this case appropriately, e.g. throwing a corresponding exception. + * + * @param rs is the ResultSet holding the data + * @param index is the column index + * @param requiredType the required value type (may be null) + * @return the value object + * @throws SQLException if thrown by the JDBC API + */ + private static String getResultSetValue(ResultSet rs, int index, Class requiredType) throws SQLException { + if (requiredType == null) { + return getResultSetValue(rs, index); + } + + Object value = null; + boolean wasNullCheck = false; + + // Explicitly extract typed value, as far as possible. + if (String.class.equals(requiredType)) { + value = rs.getString(index); + } else if (boolean.class.equals(requiredType) || Boolean.class.equals(requiredType)) { + value = Boolean.valueOf(rs.getBoolean(index)); + wasNullCheck = true; + } else if (byte.class.equals(requiredType) || Byte.class.equals(requiredType)) { + value = new Byte(rs.getByte(index)); + wasNullCheck = true; + } else if (short.class.equals(requiredType) || Short.class.equals(requiredType)) { + value = new Short(rs.getShort(index)); + wasNullCheck = true; + } else if (int.class.equals(requiredType) || Integer.class.equals(requiredType)) { + value = new Long(rs.getLong(index)); + wasNullCheck = true; + } else if (long.class.equals(requiredType) || Long.class.equals(requiredType)) { + value = rs.getBigDecimal(index); + wasNullCheck = true; + } else if (float.class.equals(requiredType) || Float.class.equals(requiredType)) { + value = new Float(rs.getFloat(index)); + wasNullCheck = true; + } else if (double.class.equals(requiredType) || Double.class.equals(requiredType) + || Number.class.equals(requiredType)) { + value = new Double(rs.getDouble(index)); + wasNullCheck = true; + } else if (Time.class.equals(requiredType)) { + // try { + // value = rs.getTime(index); + // } catch (SQLException e) { + value = rs.getString(index);// 尝试拿为string对象，0000无法用Time表示 + // if (value == null && !rs.wasNull()) { + // value = "00:00:00"; // + // mysql设置了zeroDateTimeBehavior=convertToNull，出现0值时返回为null + // } + // } + } else if (Timestamp.class.equals(requiredType) || Date.class.equals(requiredType)) { + // try { + // value = convertTimestamp(rs.getTimestamp(index)); + // } catch (SQLException e) { + // 尝试拿为string对象，0000-00-00 00:00:00无法用Timestamp 表示 + value = rs.getString(index); + // if (value == null && !rs.wasNull()) { + // value = "0000:00:00 00:00:00"; // + // mysql设置了zeroDateTimeBehavior=convertToNull，出现0值时返回为null + // } + // } + } else if (BigDecimal.class.equals(requiredType)) { + value = rs.getBigDecimal(index); + } else if (BigInteger.class.equals(requiredType)) { + value = rs.getBigDecimal(index); + } else if (Blob.class.equals(requiredType)) { + value = rs.getBlob(index); + } else if (Clob.class.equals(requiredType)) { + value = rs.getClob(index); + } else if (byte[].class.equals(requiredType)) { + try { + byte[] bytes = rs.getBytes(index); + if (bytes == null) { + value = null; + } else { + value = new String(bytes, "ISO-8859-1");// 将binary转化为iso-8859-1的字符串 + } + } catch (UnsupportedEncodingException e) { + throw new SQLException(e); + } + } else { + // Some unknown type desired -> rely on getObject. + value = getResultSetValue(rs, index); + } + + // Perform was-null check if demanded (for results that the + // JDBC driver returns as primitives). + if (wasNullCheck && (value != null) && rs.wasNull()) { + value = null; + } + + return (value == null) ? null : convertUtilsBean.convert(value); + } + + /** + * Retrieve a JDBC column value from a ResultSet, using the most appropriate + * value type. The returned value should be a detached value object, not + * having any ties to the active ResultSet: in particular, it should not be + * a Blob or Clob object but rather a byte array respectively String + * representation. + *

    + * Uses the getObject(index) method, but includes additional + * "hacks" to get around Oracle 10g returning a non-standard object for its + * TIMESTAMP datatype and a java.sql.Date for DATE columns + * leaving out the time portion: These columns will explicitly be extracted + * as standard java.sql.Timestamp object. + * + * @param rs is the ResultSet holding the data + * @param index is the column index + * @return the value object + * @throws SQLException if thrown by the JDBC API + * @see Blob + * @see Clob + * @see Timestamp + */ + private static String getResultSetValue(ResultSet rs, int index) throws SQLException { + Object obj = rs.getObject(index); + return (obj == null) ? null : convertUtilsBean.convert(obj); + } + + // private static Object convertTimestamp(Timestamp timestamp) { + // return (timestamp == null) ? null : timestamp.getTime(); + // } + + /** + * Check whether the given SQL type is numeric. + */ + public static boolean isNumeric(int sqlType) { + return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) + || (Types.DOUBLE == sqlType) || (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) + || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) || (Types.SMALLINT == sqlType) + || (Types.TINYINT == sqlType); + } + + public static boolean isTextType(int sqlType) { + if (sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR + || sqlType == Types.NCHAR || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB + || sqlType == Types.LONGNVARCHAR) { + return true; + } else { + return false; + } + } +} diff --git a/example/src/main/resources/client-spring.xml b/example/src/main/resources/client-spring.xml new file mode 100644 index 00000000..d9319007 --- /dev/null +++ b/example/src/main/resources/client-spring.xml @@ -0,0 +1,53 @@ + + + + + + + + + classpath:client.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/example/src/main/resources/client.properties b/example/src/main/resources/client.properties new file mode 100644 index 00000000..c5142912 --- /dev/null +++ b/example/src/main/resources/client.properties @@ -0,0 +1,16 @@ +# client 配置 +zk.servers=127.0.0.1:2181 +# 5 * 1024 +client.batch.size=5120 +client.debug=false +client.destination=example +client.username=canal +client.password=canal +client.exceptionstrategy=1 +client.retrytimes=3 +client.filter=.*\\..* + +# 同步目标: mysql 配置 +target.mysql.url=jdbc:mysql://127.0.0.1:4306 +target.mysql.username=root +target.mysql.password=123456 From 02868d8fecb50e445d8b624ca70b25da36cda788 Mon Sep 17 00:00:00 2001 From: yfpeng Date: Tue, 31 Jul 2018 20:30:32 +0800 Subject: [PATCH 31/37] =?UTF-8?q?=E4=B8=BA=E4=BA=86=E8=A7=A3=E5=86=B3?= =?UTF-8?q?=E5=9B=9E=E8=BF=BD=E6=95=B0=E6=8D=AE=E6=97=B6Rowdata=E4=B8=8E?= =?UTF-8?q?=E5=BD=93=E5=89=8Dschema=E4=B8=8D=E5=8C=B9=E9=85=8D=20=E5=AF=B9?= =?UTF-8?q?TableMeta=E6=8F=90=E4=BE=9B=E6=8C=81=E4=B9=85=E5=8C=96=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E5=8F=AF=E4=BB=A5=E6=A0=B9=E6=8D=AEtimestamp?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E5=8E=86=E5=8F=B2schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/CanalInstanceWithManager.java | 8 + .../manager/model/CanalParameter.java | 12 ++ .../parse/inbound/mysql/MysqlEventParser.java | 27 ++- .../inbound/mysql/dbsync/LogEventConvert.java | 11 +- .../inbound/mysql/dbsync/TableMetaCache.java | 7 +- .../tablemeta/HistoryTableMetaCache.java | 191 ++++++++++++++++++ .../tablemeta/TableMetaCacheInterface.java | 20 ++ .../tablemeta/TableMetaCacheWithStorage.java | 105 ++++++++++ .../mysql/tablemeta/TableMetaEntry.java | 55 +++++ .../mysql/tablemeta/TableMetaStorage.java | 18 ++ .../tablemeta/TableMetaStorageFactory.java | 9 + .../exception/CacheConnectionNull.java | 9 + .../exception/NoHistoryException.java | 21 ++ .../impl/mysql/MySqlTableMetaCallback.java | 14 ++ .../impl/mysql/MySqlTableMetaStorage.java | 48 +++++ .../mysql/MySqlTableMetaStorageFactory.java | 26 +++ .../mysql/tablemeta/NoStorageTest.java | 38 ++++ .../inbound/mysql/tablemeta/StorageTest.java | 77 +++++++ pom.xml | 2 +- 19 files changed, 687 insertions(+), 11 deletions(-) create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index bca5ba13..f6cec13d 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.HistoryTableMetaCache; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -242,6 +243,13 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds()); // 数据库信息参数 mysqlEventParser.setSlaveId(parameters.getSlaveId()); + mysqlEventParser.setTableMetaStorageFactory(parameters.getTableMetaStorageFactory()); + // Ctrip callback +// mysqlEventParser.setCallback(parameters.getCallback()); +// HistoryTableMetaCache cache = new HistoryTableMetaCache(); +// cache.init(parameters.getEntries()); +// mysqlEventParser.setHistoryTableMetaCache(cache); + if (!CollectionUtils.isEmpty(dbAddresses)) { mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index 16c929ca..94558590 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -5,6 +5,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.ToStringBuilder; @@ -106,6 +107,9 @@ public class CanalParameter implements Serializable { private Long standbyLogfileOffest = null; private Long standbyTimestamp = null; + // Ctrip Table Meta + TableMetaStorageFactory tableMetaStorageFactory; + public static enum RunMode { /** 嵌入式 */ @@ -859,6 +863,14 @@ public class CanalParameter implements Serializable { this.blackFilter = blackFilter; } + public TableMetaStorageFactory getTableMetaStorageFactory() { + return tableMetaStorageFactory; + } + + public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) { + this.tableMetaStorageFactory = tableMetaStorageFactory; + } + public String toString() { return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 2efab0ee..a30af259 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -10,6 +10,11 @@ import java.util.Map; import java.util.TimerTask; import java.util.concurrent.atomic.AtomicLong; +import com.alibaba.otter.canal.parse.inbound.*; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheWithStorage; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory; import org.apache.commons.lang.StringUtils; import org.springframework.util.CollectionUtils; @@ -20,9 +25,6 @@ import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.ha.CanalHAController; -import com.alibaba.otter.canal.parse.inbound.ErosaConnection; -import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback; -import com.alibaba.otter.canal.parse.inbound.SinkFunction; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogFormat; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogImage; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; @@ -62,7 +64,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // 心跳检查信息 private String detectingSQL; // 心跳sql private MysqlConnection metaConnection; // 查询meta信息的链接 - private TableMetaCache tableMetaCache; // 对应meta + private TableMetaCacheInterface tableMetaCache; // 对应meta // cache private int fallbackIntervalInSeconds = 60; // 切换回退时间 private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验 @@ -72,6 +74,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE private int dumpErrorCount = 0; // binlogDump失败异常计数 private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 + private TableMetaStorageFactory tableMetaStorageFactory; + protected ErosaConnection buildErosaConnection() { return buildMysqlConnection(this.runningInfo); } @@ -125,7 +129,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); } - tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); + + TableMetaStorage storage = null; + if (tableMetaStorageFactory != null) { + storage = tableMetaStorageFactory.getTableMetaStorage(); + } + + tableMetaCache = new TableMetaCacheWithStorage(metaConnection, storage); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } } @@ -908,4 +918,11 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE this.dumpErrorCountThreshold = dumpErrorCountThreshold; } + public TableMetaStorageFactory getTableMetaStorageFactory() { + return tableMetaStorageFactory; + } + + public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) { + this.tableMetaStorageFactory = tableMetaStorageFactory; + } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index fe34e199..67bb07b7 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -10,6 +10,9 @@ import java.util.Arrays; import java.util.BitSet; import java.util.List; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; @@ -88,7 +91,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时 private volatile AviaterRegexFilter nameBlackFilter; - private TableMetaCache tableMetaCache; + private TableMetaCacheInterface tableMetaCache; private String binlogFileName = "mysql-bin.000001"; private Charset charset = Charset.defaultCharset(); private boolean filterQueryDcl = false; @@ -262,7 +265,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar if (!isSeek) { // 使用新的表结构元数据管理方式 EntryPosition position = createPosition(event.getHeader()); - tableMetaCache.apply(position, event.getDbName(), queryString, null); + String fulltbName = schemaName+"."+tableName; + tableMetaCache.apply(position, fulltbName, queryString, null); } Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type); @@ -901,7 +905,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar this.nameBlackFilter = nameBlackFilter; } - public void setTableMetaCache(TableMetaCache tableMetaCache) { + public void setTableMetaCache(TableMetaCacheInterface tableMetaCache) { this.tableMetaCache = tableMetaCache; } @@ -928,5 +932,4 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar public void setGtidSet(GTIDSet gtidSet) { this.gtidSet = gtidSet; } - } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index 4c1cef3c..5c782aa1 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; @@ -29,7 +30,7 @@ import com.google.common.cache.LoadingCache; * @author jianghang 2013-1-17 下午10:15:16 * @version 1.0.0 */ -public class TableMetaCache { +public class TableMetaCache implements TableMetaCacheInterface { public static final String COLUMN_NAME = "COLUMN_NAME"; public static final String COLUMN_TYPE = "COLUMN_TYPE"; @@ -99,6 +100,10 @@ public class TableMetaCache { String createDDL = packet.getFieldValues().get(1); MemoryTableMeta memoryTableMeta = new MemoryTableMeta(); memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null); + String[] strings = table.split("\\."); + if (strings.length > 1) { + table = strings[1]; + } TableMeta tableMeta = memoryTableMeta.find(schema, table); return tableMeta.getFields(); } else { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java new file mode 100644 index 00000000..0964627d --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java @@ -0,0 +1,191 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.CacheConnectionNull; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + +import java.io.IOException; +import java.util.*; + +public class HistoryTableMetaCache { + private TableMetaStorage tableMetaStorage; + private MysqlConnection metaConnection; + private LoadingCache> cache; // 第一层：数据库名.表名，第二层时间戳，TableMeta + + public HistoryTableMetaCache() { + cache = CacheBuilder.newBuilder().build(new CacheLoader>() { + @Override + public Map load(String tableName) throws Exception { + Long timestamp = new Date().getTime(); + String[] strs = tableName.split("\\."); + String schema = strs[0]; + if (tableMetaStorage != null) { + init(tableMetaStorage.fetchByTableName(tableName)); // 从存储中读取表的历史ddl + } + ResultSetPacket resultSetPacket = connectionQuery("show create table " + tableName); // 获取当前ddl + String currentDdl = resultSetPacket.getFieldValues().get(1); + if (cache.asMap().containsKey(tableName)) { + Map tableMetaMap = cache.getUnchecked(tableName); + if (tableMetaMap.isEmpty()) { + put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema，取时间为当前时间-1s + } else { // 如果table存在历史 + Iterator iterator = tableMetaMap.keySet().iterator(); + Long firstTimestamp = iterator.next(); + TableMeta first = tableMetaMap.get(firstTimestamp); // 拿第一条ddl + if (!first.getDdl().equalsIgnoreCase(currentDdl)) { // 当前ddl与历史第一条不一致，放入当前ddl + put(schema, tableName, currentDdl, calculateNewTimestamp(firstTimestamp)); // 计算放入的timestamp,设为第一条时间+1s + } + } + } else { + put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema + } + return cache.get(tableName); + } + }); + } + + public void init(List entries) throws IOException { + if (entries == null) { + return; + } + for (TableMetaEntry entry : entries) { + try { + put(entry.getSchema(), entry.getTable(), entry.getDdl(), entry.getTimestamp()); + } catch (CacheConnectionNull cacheConnectionNull) { + cacheConnectionNull.printStackTrace(); + } + } + } + + public TableMeta put(String schema, String table, String ddl, Long timestamp) throws CacheConnectionNull, IOException { + ResultSetPacket resultSetPacket; + if (!(ddl.contains("CREATE TABLE") || ddl.contains("create table"))) { // 尝试直接从数据库拉取CREATE TABLE的DDL + resultSetPacket = connectionQuery("show create table " + table); + ddl = resultSetPacket.getFieldValues().get(1); + } else { // CREATE TABLE 的 DDL + resultSetPacket = new ResultSetPacket(); + List fields = new ArrayList(); + String[] strings = table.split("\\."); + String shortTable = table; + if (strings.length > 1) { + shortTable = strings[1]; + } + fields.add(0, shortTable); + fields.add(1, ddl); + resultSetPacket.setFieldValues(fields); + if (metaConnection != null) { + resultSetPacket.setSourceAddress(metaConnection.getAddress()); + } + } + Map tableMetaMap; + if (!cache.asMap().containsKey(table)) { + tableMetaMap = new TreeMap(new Comparator() { + @Override + public int compare(Long o1, Long o2) { + return o2.compareTo(o1); + } + }); + cache.put(table, tableMetaMap); + } else { + tableMetaMap = cache.getUnchecked(table); + } + eliminate(tableMetaMap); // 淘汰旧的TableMeta + TableMeta tableMeta = new TableMeta(schema, table, TableMetaCache.parseTableMeta(schema, table, resultSetPacket)); + if (tableMeta.getDdl() == null) { // 生成的TableMeta有时DDL为null + tableMeta.setDdl(ddl); + } + tableMetaMap.put(timestamp, tableMeta); + return tableMeta; + } + + public TableMeta get(String schema, String table, Long timestamp) throws NoHistoryException, CacheConnectionNull { + Map tableMetaMap = cache.getUnchecked(table); + Iterator iterator = tableMetaMap.keySet().iterator(); + Long selected = null; + while(iterator.hasNext()) { + Long temp = iterator.next(); + if (timestamp > temp) { + selected = temp; + break; + } + } + + if (selected == null) { + iterator = tableMetaMap.keySet().iterator(); + if (iterator.hasNext()) { + selected = iterator.next(); + } else { + throw new NoHistoryException(schema, table); + } + } + + return tableMetaMap.get(selected); + } + + public void clearTableMeta() { + cache.invalidateAll(); + } + + public void clearTableMetaWithSchemaName(String schema) { + for (String tableName : cache.asMap().keySet()) { + String[] strs = tableName.split("\\."); + if (schema.equalsIgnoreCase(strs[0])) { + cache.invalidate(tableName); + } + } + } + + public void clearTableMeta(String schema, String table) { + if (!table.contains(".")) { + table = schema+"."+table; + } + cache.invalidate(table); + } + + // eliminate older table meta in cache + private void eliminate(Map tableMetaMap) { + int MAX_CAPABILITY = 20; + if (tableMetaMap.keySet().size() < MAX_CAPABILITY) { + return; + } + Iterator iterator = tableMetaMap.keySet().iterator(); + while(iterator.hasNext()) { + iterator.next(); + } + iterator.remove(); + } + + private Long calculateNewTimestamp(Long oldTimestamp) { + return oldTimestamp + 1000; + } + + private ResultSetPacket connectionQuery(String query) throws CacheConnectionNull, IOException { + if (metaConnection == null) { + throw new CacheConnectionNull(); + } + try { + return metaConnection.query(query); + } catch (IOException e) { + try { + metaConnection.reconnect(); + return metaConnection.query(query); + } catch (IOException e1) { + throw e1; + } + } + } + + public void setMetaConnection(MysqlConnection metaConnection) { + this.metaConnection = metaConnection; + } + + public void setTableMetaStorage(TableMetaStorage tableMetaStorage) { + this.tableMetaStorage = tableMetaStorage; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java new file mode 100644 index 00000000..5196ee08 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java @@ -0,0 +1,20 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.protocol.position.EntryPosition; + +public interface TableMetaCacheInterface { + + TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position); + + void clearTableMeta(); + + void clearTableMetaWithSchemaName(String schema); + + void clearTableMeta(String schema, String table); + + boolean apply(EntryPosition position, String schema, String ddl, String extra); + + boolean isOnRDS(); + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java new file mode 100644 index 00000000..eb43158b --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java @@ -0,0 +1,105 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.protocol.position.EntryPosition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; + +public class TableMetaCacheWithStorage implements TableMetaCacheInterface { + + private static Logger logger = LoggerFactory.getLogger(TableMetaCacheWithStorage.class); + private TableMetaStorage tableMetaStorage; // TableMeta存储 + private HistoryTableMetaCache cache = new HistoryTableMetaCache(); // cache + + public TableMetaCacheWithStorage(MysqlConnection con, TableMetaStorage tableMetaStorage) { + this.tableMetaStorage = tableMetaStorage; + InetSocketAddress address = con.getAddress(); + this.tableMetaStorage.setDbAddress(address.getHostName()+":"+address.getPort()); + cache.setMetaConnection(con); + cache.setTableMetaStorage(tableMetaStorage); + if (tableMetaStorage != null) { + try { + cache.init(tableMetaStorage.fetch()); // 初始化，从存储拉取TableMeta + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + } + + @Override + public boolean apply(EntryPosition position, String fullTableName, String ddl, String extra) { + String[] strs = fullTableName.split("\\."); + String schema = strs[0]; + if (schema.equalsIgnoreCase("null")) { // ddl schema为null，放弃处理 + return false; + } + try { + TableMeta tableMeta = cache.get(schema, fullTableName, position.getTimestamp()); + if (!compare(tableMeta, ddl)) { // 获取最近的TableMeta，进行比对 + TableMeta result = cache.put(schema, fullTableName, ddl, calTimestamp(position.getTimestamp())); + if (tableMetaStorage != null && result != null) { // 储存 + tableMetaStorage.store(schema, fullTableName, result.getDdl(), calTimestamp(position.getTimestamp())); + } + } + return true; + } catch (Exception e) { + logger.error(e.toString()); + } + + return false; + } + + @Override + public boolean isOnRDS() { + return false; + } + + /*** + * + * @param schema dbname + * @param table tablename + * @param useCache unused + * @param position timestamp + * @return + */ + @Override + public TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) { + String fulltbName = schema + "." + table; + try { + return cache.get(schema, fulltbName, position.getTimestamp()); + } catch (Exception e) { + logger.error(e.toString()); + } + return null; + } + + @Override + public void clearTableMeta() { + cache.clearTableMeta(); + } + + @Override + public void clearTableMetaWithSchemaName(String schema) { + cache.clearTableMetaWithSchemaName(schema); + } + + @Override + public void clearTableMeta(String schema, String table) { + cache.clearTableMeta(schema, table); + } + + private boolean compare(TableMeta tableMeta, String ddl) { + if (tableMeta == null) { + return false; + } + return tableMeta.getDdl().equalsIgnoreCase(ddl); + } + + private Long calTimestamp(Long timestamp) { + return timestamp; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java new file mode 100644 index 00000000..03e63ed7 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java @@ -0,0 +1,55 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import java.io.Serializable; + +public class TableMetaEntry implements Serializable { + + private static final long serialVersionUID = -1350200637109107904L; + + private String dbAddress; + private String schema; + private String table; + private String ddl; + private Long timestamp; + + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getDdl() { + return ddl; + } + + public void setDdl(String ddl) { + this.ddl = ddl; + } + + public Long getTimestamp() { + return timestamp; + } + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + + public String getDbAddress() { + return dbAddress; + } + + public void setDbAddress(String dbAddress) { + this.dbAddress = dbAddress; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java new file mode 100644 index 00000000..ac9d7a0a --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import java.util.List; + +public interface TableMetaStorage { + + void store(String schema, String table, String ddl, Long timestamp); + + List fetch(); + + List fetchByTableName(String tableName); + + String getDbName(); + + String getDbAddress(); + + void setDbAddress(String address); +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java new file mode 100644 index 00000000..0e532ebb --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java @@ -0,0 +1,9 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +public interface TableMetaStorageFactory { + + TableMetaStorage getTableMetaStorage(); + + String getDbName(); + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java new file mode 100644 index 00000000..1dadd78a --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java @@ -0,0 +1,9 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception; + +public class CacheConnectionNull extends Exception{ + + @Override + public String toString() { + return "CacheConnectionNull"; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java new file mode 100644 index 00000000..d8f1d16b --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java @@ -0,0 +1,21 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception; + +public class NoHistoryException extends Exception{ + + private String dbName; + private String tbName; + + public NoHistoryException(String dbName, String tbName) { + this.dbName = dbName; + this.tbName = tbName; + } + + public void printTableName() { + System.out.println(dbName+"."+tbName); + } + + @Override + public String toString() { + return "NioHistoryException: " + dbName + " " + tbName; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java new file mode 100644 index 00000000..661986a1 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java @@ -0,0 +1,14 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql; + +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry; + +import java.util.List; + +public interface MySqlTableMetaCallback { + + void save(String dbAddress, String schema, String table,String ddl, Long timestamp); + + List fetch(String dbAddress, String dbName); + + List fetch(String dbAddress, String dbName, String tableName); +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java new file mode 100644 index 00000000..f120ef80 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java @@ -0,0 +1,48 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql; + +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; + +import java.util.List; + +public class MySqlTableMetaStorage implements TableMetaStorage { + private MySqlTableMetaCallback mySqlTableMetaCallback; + private String dbName; + private String dbAddress; + + MySqlTableMetaStorage(MySqlTableMetaCallback callback, String dbName) { + mySqlTableMetaCallback = callback; + this.dbName = dbName; + } + + + @Override + public void store(String schema, String table, String ddl, Long timestamp) { + mySqlTableMetaCallback.save(dbAddress, schema, table, ddl, timestamp); + } + + @Override + public List fetch() { + return mySqlTableMetaCallback.fetch(dbAddress, dbName); + } + + @Override + public List fetchByTableName(String tableName) { + return mySqlTableMetaCallback.fetch(dbAddress, dbName, tableName); + } + + @Override + public String getDbName() { + return dbName; + } + + @Override + public String getDbAddress() { + return dbAddress; + } + + @Override + public void setDbAddress(String address) { + this.dbAddress = address; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java new file mode 100644 index 00000000..0eee235d --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java @@ -0,0 +1,26 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql; + +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory; + +public class MySqlTableMetaStorageFactory implements TableMetaStorageFactory { + + private MySqlTableMetaCallback mySQLTableMetaCallback; + private String dbName; + + public MySqlTableMetaStorageFactory(MySqlTableMetaCallback callback, String dbName) { + mySQLTableMetaCallback = callback; + this.dbName = dbName; + } + + @Override + public TableMetaStorage getTableMetaStorage() { + return new MySqlTableMetaStorage(mySQLTableMetaCallback, dbName); + } + + @Override + public String getDbName() { + return dbName; + } + +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java new file mode 100644 index 00000000..04fd95ea --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.protocol.position.EntryPosition; +import org.junit.Test; + +import java.net.InetSocketAddress; +import java.util.Date; + +public class NoStorageTest { + final String DBNAME = "testdb"; + final String TBNAME = "testtb"; + final String DDL = "CREATE TABLE `testtb` (\n" + + " `id` int(11) NOT NULL AUTO_INCREMENT,\n" + + " `name` varchar(2048) DEFAULT NULL,\n" + + " `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" + + " `otter_testcol` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol1` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol2` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol3` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol4` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol5` varchar(45) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + " ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4"; + @Test + public void nostorage() { + MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"); + TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, null); + EntryPosition entryPosition = new EntryPosition(); + entryPosition.setTimestamp(new Date().getTime()); + String fullTableName = DBNAME + "." + TBNAME; + tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null); + entryPosition.setTimestamp(new Date().getTime() + 1000L); + TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition); + assert result.getDdl().equalsIgnoreCase(DDL); + } +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java new file mode 100644 index 00000000..0ec16607 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java @@ -0,0 +1,77 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback; +import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory; +import com.alibaba.otter.canal.protocol.position.EntryPosition; +import com.alibaba.otter.canal.protocol.position.Position; +import org.junit.Test; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class StorageTest { + + final String DBNAME = "testdb"; + final String TBNAME = "testtb"; + final String DDL = "CREATE TABLE `testtb` (\n" + + " `id` int(11) NOT NULL AUTO_INCREMENT,\n" + + " `name` varchar(2048) DEFAULT NULL,\n" + + " `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" + + " `otter_testcol` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol1` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol2` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol3` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol4` varchar(45) DEFAULT NULL,\n" + + " `otter_testcol5` varchar(45) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + " ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4"; + + @Test + public void storage() { + + MySqlTableMetaStorageFactory factory = new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() { + @Override + public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) { + + } + + @Override + public List fetch(String dbAddress, String dbName) { + TableMetaEntry tableMeta = new TableMetaEntry(); + tableMeta.setSchema(DBNAME); + tableMeta.setTable(TBNAME); + tableMeta.setDdl(DDL); + tableMeta.setTimestamp(new Date().getTime()); + List entries = new ArrayList(); + entries.add(tableMeta); + return entries; + } + + @Override + public List fetch(String dbAddress, String dbName, String tableName) { + TableMetaEntry tableMeta = new TableMetaEntry(); + tableMeta.setSchema(DBNAME); + tableMeta.setTable(TBNAME); + tableMeta.setDdl(DDL); + tableMeta.setTimestamp(new Date().getTime()); + List entries = new ArrayList(); + entries.add(tableMeta); + return entries; + } + }, DBNAME); + MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"); + TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, factory.getTableMetaStorage()); + EntryPosition entryPosition = new EntryPosition(); + entryPosition.setTimestamp(new Date().getTime()); + String fullTableName = DBNAME + "." + TBNAME; + tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null); + + entryPosition.setTimestamp(new Date().getTime() + 1000L); + TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition); + assert result.getDdl().equalsIgnoreCase(DDL); + } +} diff --git a/pom.xml b/pom.xml index f48a488a..e9c2623b 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_366 + 2.0.0_preview_186 com.alibaba From cd0bf60824e6e469982c2a7b04448393c5198529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Tue, 7 Aug 2018 16:19:58 +0800 Subject: [PATCH 32/37] fixed compiler java 1.7 --- example/pom.xml | 13 ------------- pom.xml | 6 +++--- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/example/pom.xml b/example/pom.xml index ba7a6dde..78e48c4b 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -21,11 +21,6 @@ canal.protocol ${project.version} - - com.alibaba.otter - canal.instance.spring - ${project.version} - org.apache.ddlutils ddlutils @@ -135,14 +130,6 @@ false - - org.apache.maven.plugins - maven-compiler-plugin - - 7 - 7 - - diff --git a/pom.xml b/pom.xml index 9a14868b..24fe92db 100644 --- a/pom.xml +++ b/pom.xml @@ -96,8 +96,8 @@ true true - 1.6 - 1.6 + 1.7 + 1.7 UTF-8 3.2.9.RELEASE @@ -332,7 +332,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.2 + 3.8.0 ${java_source_version} ${java_target_version} From 4006cedaad464e2b2accd6d92b7401a4cbf6d643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Tue, 7 Aug 2018 16:44:40 +0800 Subject: [PATCH 33/37] =?UTF-8?q?fixed=20issue=20#781,=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E9=83=A8=E5=88=86DDL=E8=A7=A3=E6=9E=90=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 24fe92db..1e4bdbf6 100644 --- a/pom.xml +++ b/pom.xml @@ -247,7 +247,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_520 + 2.0.0_preview_538 com.alibaba From 6d3584b6fb0f8060427ecf23a64776e27c50edad Mon Sep 17 00:00:00 2001 From: winger Date: Tue, 7 Aug 2018 16:59:55 +0800 Subject: [PATCH 34/37] =?UTF-8?q?=E7=A7=BB=E9=99=A4canal.instance.spring?= =?UTF-8?q?=20=E4=BE=9D=E8=B5=96=E5=90=8E=EF=BC=8C=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E5=8A=A0=E5=85=A5druid=E4=B8=8Emysql=20connector=E7=9A=84?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=202.=20PropertyPlaceholderConfigurer?= =?UTF-8?q?=E7=94=B1canal.instance.spring=E7=A7=BB=E8=87=B3db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- example/pom.xml | 8 + .../db/PropertyPlaceholderConfigurer.java | 169 ++++++++++++++++++ example/src/main/resources/client-spring.xml | 2 +- 3 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/db/PropertyPlaceholderConfigurer.java diff --git a/example/pom.xml b/example/pom.xml index 78e48c4b..2897e821 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -21,6 +21,14 @@ canal.protocol ${project.version} + + com.alibaba + druid + + + mysql + mysql-connector-java + org.apache.ddlutils ddlutils diff --git a/example/src/main/java/com/alibaba/otter/canal/example/db/PropertyPlaceholderConfigurer.java b/example/src/main/java/com/alibaba/otter/canal/example/db/PropertyPlaceholderConfigurer.java new file mode 100644 index 00000000..65b4827c --- /dev/null +++ b/example/src/main/java/com/alibaba/otter/canal/example/db/PropertyPlaceholderConfigurer.java @@ -0,0 +1,169 @@ +package com.alibaba.otter.canal.example.db; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.context.ResourceLoaderAware; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * 扩展Spring的 + * {@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer} + * ，增加默认值的功能。 例如：${placeholder:defaultValue}，假如placeholder的值不存在，则默认取得 + * defaultValue。 + * + * @author jianghang 2013-1-24 下午03:37:56 + * @version 1.0.0 + */ +public class PropertyPlaceholderConfigurer extends org.springframework.beans.factory.config.PropertyPlaceholderConfigurer implements ResourceLoaderAware, InitializingBean { + + private static final String PLACEHOLDER_PREFIX = "${"; + private static final String PLACEHOLDER_SUFFIX = "}"; + private ResourceLoader loader; + private String[] locationNames; + + public PropertyPlaceholderConfigurer(){ + setIgnoreUnresolvablePlaceholders(true); + } + + public void setResourceLoader(ResourceLoader loader) { + this.loader = loader; + } + + public void setLocationNames(String[] locations) { + this.locationNames = locations; + } + + public void afterPropertiesSet() throws Exception { + Assert.notNull(loader, "no resourceLoader"); + + if (locationNames != null) { + for (int i = 0; i < locationNames.length; i++) { + locationNames[i] = resolveSystemPropertyPlaceholders(locationNames[i]); + } + } + + if (locationNames != null) { + List resources = new ArrayList(locationNames.length); + + for (String location : locationNames) { + location = trimToNull(location); + + if (location != null) { + resources.add(loader.getResource(location)); + } + } + + super.setLocations(resources.toArray(new Resource[resources.size()])); + } + } + + private String resolveSystemPropertyPlaceholders(String text) { + StringBuilder buf = new StringBuilder(text); + + for (int startIndex = buf.indexOf(PLACEHOLDER_PREFIX); startIndex >= 0;) { + int endIndex = buf.indexOf(PLACEHOLDER_SUFFIX, startIndex + PLACEHOLDER_PREFIX.length()); + + if (endIndex != -1) { + String placeholder = buf.substring(startIndex + PLACEHOLDER_PREFIX.length(), endIndex); + int nextIndex = endIndex + PLACEHOLDER_SUFFIX.length(); + + try { + String value = resolveSystemPropertyPlaceholder(placeholder); + + if (value != null) { + buf.replace(startIndex, endIndex + PLACEHOLDER_SUFFIX.length(), value); + nextIndex = startIndex + value.length(); + } else { + System.err.println("Could not resolve placeholder '" + + placeholder + + "' in [" + + text + + "] as system property: neither system property nor environment variable found"); + } + } catch (Throwable ex) { + System.err.println("Could not resolve placeholder '" + placeholder + "' in [" + text + + "] as system property: " + ex); + } + + startIndex = buf.indexOf(PLACEHOLDER_PREFIX, nextIndex); + } else { + startIndex = -1; + } + } + + return buf.toString(); + } + + private String resolveSystemPropertyPlaceholder(String placeholder) { + DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder); + String value = System.getProperty(dp.placeholder); + + if (value == null) { + value = System.getenv(dp.placeholder); + } + + if (value == null) { + value = dp.defaultValue; + } + + return value; + } + + @Override + protected String resolvePlaceholder(String placeholder, Properties props, int systemPropertiesMode) { + DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder); + String value = super.resolvePlaceholder(dp.placeholder, props, systemPropertiesMode); + + if (value == null) { + value = dp.defaultValue; + } + + return trimToEmpty(value); + } + + private static class DefaultablePlaceholder { + + private final String defaultValue; + private final String placeholder; + + public DefaultablePlaceholder(String placeholder){ + int commaIndex = placeholder.indexOf(":"); + String defaultValue = null; + + if (commaIndex >= 0) { + defaultValue = trimToEmpty(placeholder.substring(commaIndex + 1)); + placeholder = trimToEmpty(placeholder.substring(0, commaIndex)); + } + + this.placeholder = placeholder; + this.defaultValue = defaultValue; + } + } + + private String trimToNull(String str) { + if (str == null) { + return null; + } + + String result = str.trim(); + + if (result == null || result.length() == 0) { + return null; + } + + return result; + } + + public static String trimToEmpty(String str) { + if (str == null) { + return ""; + } + + return str.trim(); + } +} diff --git a/example/src/main/resources/client-spring.xml b/example/src/main/resources/client-spring.xml index d9319007..29eba1af 100644 --- a/example/src/main/resources/client-spring.xml +++ b/example/src/main/resources/client-spring.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd" default-autowire="byName"> - + From f36ce647e690349bb710c01287f359274dd656c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Tue, 7 Aug 2018 17:46:36 +0800 Subject: [PATCH 35/37] fixed issue #781 , upgrade fastsql 540 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1e4bdbf6..56ac821a 100644 --- a/pom.xml +++ b/pom.xml @@ -247,7 +247,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_538 + 2.0.0_preview_540 com.alibaba From 23f7b3b5d82c3571454877e32bc6f5bbd020b4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Tue, 7 Aug 2018 18:51:23 +0800 Subject: [PATCH 36/37] fixed issue #818 , tableMeta tsdb bugfix --- .../otter/canal/deployer/CanalController.java | 3 ++- .../inbound/mysql/AbstractMysqlEventParser.java | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java index 0a2c91f8..6b0fc934 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java @@ -34,6 +34,7 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; import com.alibaba.otter.canal.instance.manager.CanalConfigClient; import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator; import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator; +import com.alibaba.otter.canal.parse.CanalEventParser; import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; import com.alibaba.otter.canal.server.exception.CanalServerException; import com.alibaba.otter.canal.server.netty.CanalServerWithNetty; @@ -303,7 +304,7 @@ public class CanalController { return instanceGenerator.generate(destination); } else if (config.getMode().isSpring()) { SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator(); - synchronized (this) { + synchronized (CanalEventParser.class) { try { // 设置当前正在加载的通道，加载spring查找文件时会用到该变量 System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 8188cf83..c0d2f190 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.filter.CanalEventFilter; import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; +import com.alibaba.otter.canal.parse.CanalEventParser; import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet; import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; @@ -91,8 +92,16 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { public void start() throws CanalParseException { if (enableTsdb) { if (tableMetaTSDB == null) { - // 初始化 - tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + synchronized (CanalEventParser.class) { + try { + // 设置当前正在加载的通道，加载spring查找文件时会用到该变量 + System.setProperty("canal.instance.destination", destination); + // 初始化 + tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + } finally { + System.setProperty("canal.instance.destination", ""); + } + } } } From 9e816bc48f9955f9e2839873993cef9627c2f389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 8 Aug 2018 18:54:57 +0800 Subject: [PATCH 37/37] support otter use tableMetaTSDB --- .../dbsync/binlog/event/RowsLogBuffer.java | 23 ++- .../manager/CanalInstanceWithManager.java | 30 +-- .../manager/model/CanalParameter.java | 44 +++- .../mysql/AbstractMysqlEventParser.java | 43 ++-- .../parse/inbound/mysql/MysqlEventParser.java | 28 +-- .../inbound/mysql/dbsync/LogEventConvert.java | 25 +-- .../inbound/mysql/dbsync/TableMetaCache.java | 7 +- .../tablemeta/HistoryTableMetaCache.java | 191 ------------------ .../tablemeta/TableMetaCacheInterface.java | 20 -- .../tablemeta/TableMetaCacheWithStorage.java | 105 ---------- .../mysql/tablemeta/TableMetaEntry.java | 55 ----- .../mysql/tablemeta/TableMetaStorage.java | 18 -- .../tablemeta/TableMetaStorageFactory.java | 9 - .../exception/CacheConnectionNull.java | 9 - .../exception/NoHistoryException.java | 21 -- .../impl/mysql/MySqlTableMetaCallback.java | 14 -- .../impl/mysql/MySqlTableMetaStorage.java | 48 ----- .../mysql/MySqlTableMetaStorageFactory.java | 26 --- .../tsdb/DefaultTableMetaTSDBFactory.java | 19 ++ .../mysql/tsdb/TableMetaTSDBBuilder.java | 7 +- .../mysql/tsdb/TableMetaTSDBFactory.java | 18 ++ 21 files changed, 149 insertions(+), 611 deletions(-) delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DefaultTableMetaTSDBFactory.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBFactory.java diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java index fdfb3ebc..bc0f3dde 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java @@ -490,7 +490,6 @@ public final class RowsLogBuffer { // t % 100); StringBuilder builder = new StringBuilder(); - builder.append(26); appendNumber4(builder, d / 10000); builder.append('-'); appendNumber2(builder, (d % 10000) / 100); @@ -615,7 +614,13 @@ public final class RowsLogBuffer { if (i32 < 0) { builder.append('-'); } - appendNumber2(builder, u32 / 10000); + + int d = u32 / 10000; + if (d > 100) { + builder.append(String.valueOf(d)); + } else { + appendNumber2(builder, d); + } builder.append(':'); appendNumber2(builder, (u32 % 10000) / 100); builder.append(':'); @@ -724,7 +729,13 @@ public final class RowsLogBuffer { if (ltime < 0) { builder.append('-'); } - appendNumber2(builder, (int) ((intpart >> 12) % (1 << 10))); + + int d = (int) ((intpart >> 12) % (1 << 10)); + if (d > 100) { + builder.append(String.valueOf(d)); + } else { + appendNumber2(builder, d); + } builder.append(':'); appendNumber2(builder, (int) ((intpart >> 6) % (1 << 6))); builder.append(':'); @@ -1134,7 +1145,7 @@ public final class RowsLogBuffer { .append(digits[(d / 100) % 10]) .append(digits[(d / 10) % 10]) .append(digits[d % 10]); - } else if (d >= 100) { + } else { builder.append('0'); appendNumber3(builder, d); } @@ -1142,9 +1153,7 @@ public final class RowsLogBuffer { private void appendNumber3(StringBuilder builder, int d) { if (d >= 100) { - builder.append(digits[d / 100]) - .append(digits[(d / 10) % 10]) - .append(digits[d % 10]); + builder.append(digits[d / 100]).append(digits[(d / 10) % 10]).append(digits[d % 10]); } else { builder.append('0'); appendNumber2(builder, d); diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index 039a80b7..63ed2d02 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -6,11 +6,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; - -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.HistoryTableMetaCache; - -import com.alibaba.otter.canal.meta.FileMixedMetaManager; - import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +20,14 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.instance.core.AbstractCanalInstance; import com.alibaba.otter.canal.instance.manager.model.Canal; import com.alibaba.otter.canal.instance.manager.model.CanalParameter; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode; +import com.alibaba.otter.canal.meta.FileMixedMetaManager; import com.alibaba.otter.canal.meta.MemoryMetaManager; import com.alibaba.otter.canal.meta.PeriodMixedMetaManager; import com.alibaba.otter.canal.meta.ZooKeeperMetaManager; @@ -36,7 +38,12 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import com.alibaba.otter.canal.parse.index.*; +import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; +import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager; +import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager; +import com.alibaba.otter.canal.parse.index.MetaLogPositionManager; +import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager; +import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.sink.entry.EntryEventSink; @@ -114,7 +121,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(getZkclientx()); ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager); - } else if (mode.isLocalFile()){ + } else if (mode.isLocalFile()) { FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager(); fileMixedMetaManager.setDataDir(parameters.getDataDir()); fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod()); @@ -241,13 +248,6 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds()); // 数据库信息参数 mysqlEventParser.setSlaveId(parameters.getSlaveId()); - mysqlEventParser.setTableMetaStorageFactory(parameters.getTableMetaStorageFactory()); - // Ctrip callback -// mysqlEventParser.setCallback(parameters.getCallback()); -// HistoryTableMetaCache cache = new HistoryTableMetaCache(); -// cache.init(parameters.getEntries()); -// mysqlEventParser.setHistoryTableMetaCache(cache); - if (!CollectionUtils.isEmpty(dbAddresses)) { mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index d707c679..40df1485 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -5,7 +5,6 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.ToStringBuilder; @@ -94,6 +93,10 @@ public class CanalParameter implements Serializable { private Boolean filterTableError = Boolean.FALSE; // 是否忽略表解析异常 private String blackFilter = null; // 匹配黑名单,忽略解析 + private Boolean tsdbEnable = Boolean.FALSE; // 是否开启tableMetaTSDB + private String tsdbJdbcUrl; + private String tsdbJdbcUserName; + private String tsdbJdbcPassword; // ================================== 兼容字段处理 private InetSocketAddress masterAddress; // 主库信息 private String masterUsername; // 帐号 @@ -109,9 +112,6 @@ public class CanalParameter implements Serializable { private Long standbyLogfileOffest = null; private Long standbyTimestamp = null; - // Ctrip Table Meta - TableMetaStorageFactory tableMetaStorageFactory; - public static enum RunMode { /** 嵌入式 */ @@ -250,7 +250,7 @@ public class CanalParameter implements Serializable { ZOOKEEPER, /** 混合模式，内存+文件 */ MIXED, - /** 本地文件存储模式*/ + /** 本地文件存储模式 */ LOCAL_FILE; public boolean isMemory() { @@ -265,7 +265,7 @@ public class CanalParameter implements Serializable { return this.equals(MetaMode.MIXED); } - public boolean isLocalFile(){ + public boolean isLocalFile() { return this.equals(MetaMode.LOCAL_FILE); } } @@ -887,12 +887,36 @@ public class CanalParameter implements Serializable { this.blackFilter = blackFilter; } - public TableMetaStorageFactory getTableMetaStorageFactory() { - return tableMetaStorageFactory; + public Boolean getTsdbEnable() { + return tsdbEnable; } - public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) { - this.tableMetaStorageFactory = tableMetaStorageFactory; + public void setTsdbEnable(Boolean tsdbEnable) { + this.tsdbEnable = tsdbEnable; + } + + public String getTsdbJdbcUrl() { + return tsdbJdbcUrl; + } + + public void setTsdbJdbcUrl(String tsdbJdbcUrl) { + this.tsdbJdbcUrl = tsdbJdbcUrl; + } + + public String getTsdbJdbcUserName() { + return tsdbJdbcUserName; + } + + public void setTsdbJdbcUserName(String tsdbJdbcUserName) { + this.tsdbJdbcUserName = tsdbJdbcUserName; + } + + public String getTsdbJdbcPassword() { + return tsdbJdbcPassword; + } + + public void setTsdbJdbcPassword(String tsdbJdbcPassword) { + this.tsdbJdbcPassword = tsdbJdbcPassword; } public String toString() { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index c0d2f190..17eb08f4 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -14,27 +14,30 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.BinlogParser; import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DefaultTableMetaTSDBFactory; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBBuilder; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBFactory; import com.alibaba.otter.canal.protocol.position.EntryPosition; public abstract class AbstractMysqlEventParser extends AbstractEventParser { - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - protected static final long BINLOG_START_OFFEST = 4L; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + protected static final long BINLOG_START_OFFEST = 4L; + + protected TableMetaTSDBFactory tableMetaTSDBFactory = new DefaultTableMetaTSDBFactory(); + protected boolean enableTsdb = false; + protected String tsdbSpringXml; + protected TableMetaTSDB tableMetaTSDB; - protected boolean enableTsdb = false; - protected String tsdbSpringXml; - protected TableMetaTSDB tableMetaTSDB; // 编码信息 - protected byte connectionCharsetNumber = (byte) 33; - protected Charset connectionCharset = Charset.forName("UTF-8"); - protected boolean filterQueryDcl = false; - protected boolean filterQueryDml = false; - protected boolean filterQueryDdl = false; - protected boolean filterRows = false; - protected boolean filterTableError = false; - protected boolean useDruidDdlFilter = true; + protected byte connectionCharsetNumber = (byte) 33; + protected Charset connectionCharset = Charset.forName("UTF-8"); + protected boolean filterQueryDcl = false; + protected boolean filterQueryDml = false; + protected boolean filterQueryDdl = false; + protected boolean filterRows = false; + protected boolean filterTableError = false; + protected boolean useDruidDdlFilter = true; protected BinlogParser buildParser() { LogEventConvert convert = new LogEventConvert(); @@ -97,7 +100,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { // 设置当前正在加载的通道，加载spring查找文件时会用到该变量 System.setProperty("canal.instance.destination", destination); // 初始化 - tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml); } finally { System.setProperty("canal.instance.destination", ""); } @@ -110,7 +113,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { public void stop() throws CanalParseException { if (enableTsdb) { - TableMetaTSDBBuilder.destory(destination); + tableMetaTSDBFactory.destory(destination); tableMetaTSDB = null; } @@ -182,7 +185,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { if (this.enableTsdb) { if (tableMetaTSDB == null) { // 初始化 - tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml); } } } @@ -192,9 +195,13 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { if (this.enableTsdb) { if (tableMetaTSDB == null) { // 初始化 - tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml); } } } + public void setTableMetaTSDBFactory(TableMetaTSDBFactory tableMetaTSDBFactory) { + this.tableMetaTSDBFactory = tableMetaTSDBFactory; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index aeb3b2ea..53e4e5e5 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -10,11 +10,6 @@ import java.util.Map; import java.util.TimerTask; import java.util.concurrent.atomic.AtomicLong; -import com.alibaba.otter.canal.parse.inbound.*; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheWithStorage; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory; import org.apache.commons.lang.StringUtils; import org.springframework.util.CollectionUtils; @@ -25,6 +20,9 @@ import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.ha.CanalHAController; +import com.alibaba.otter.canal.parse.inbound.ErosaConnection; +import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback; +import com.alibaba.otter.canal.parse.inbound.SinkFunction; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogFormat; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogImage; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; @@ -64,8 +62,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // 心跳检查信息 private String detectingSQL; // 心跳sql private MysqlConnection metaConnection; // 查询meta信息的链接 - private TableMetaCacheInterface tableMetaCache; // 对应meta - // cache + private TableMetaCache tableMetaCache; // 对应meta private int fallbackIntervalInSeconds = 60; // 切换回退时间 private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验 private BinlogImage[] supportBinlogImages; // 支持的binlogImage,如果设置会执行强校验 @@ -74,8 +71,6 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE private int dumpErrorCount = 0; // binlogDump失败异常计数 private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 - private TableMetaStorageFactory tableMetaStorageFactory; - protected ErosaConnection buildErosaConnection() { return buildMysqlConnection(this.runningInfo); } @@ -129,13 +124,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); } - - TableMetaStorage storage = null; - if (tableMetaStorageFactory != null) { - storage = tableMetaStorageFactory.getTableMetaStorage(); - } - - tableMetaCache = new TableMetaCacheWithStorage(metaConnection, storage); + tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } } @@ -921,11 +910,4 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE this.dumpErrorCountThreshold = dumpErrorCountThreshold; } - public TableMetaStorageFactory getTableMetaStorageFactory() { - return tableMetaStorageFactory; - } - - public void setTableMetaStorageFactory(TableMetaStorageFactory tableMetaStorageFactory) { - this.tableMetaStorageFactory = tableMetaStorageFactory; - } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 72369f07..dc8e57d7 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -10,9 +10,6 @@ import java.util.Arrays; import java.util.BitSet; import java.util.List; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; @@ -90,10 +87,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时 private volatile AviaterRegexFilter nameBlackFilter; - - private TableMetaCacheInterface tableMetaCache; - private String binlogFileName = "mysql-bin.000001"; - + private TableMetaCache tableMetaCache; private Charset charset = Charset.defaultCharset(); private boolean filterQueryDcl = false; private boolean filterQueryDml = false; @@ -268,8 +262,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar if (!isSeek) { // 使用新的表结构元数据管理方式 EntryPosition position = createPosition(event.getHeader()); - String fulltbName = schemaName+"."+tableName; - tableMetaCache.apply(position, fulltbName, queryString, null); + tableMetaCache.apply(position, event.getDbName(), queryString, null); } Header header = createHeader(event.getHeader(), schemaName, tableName, type); @@ -548,12 +541,16 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta); } - rowsCount ++; + rowsCount++; rowChangeBuider.addRowDatas(rowDataBuilder.build()); } TableMapLogEvent table = event.getTable(); - Header header = createHeader(event.getHeader(), table.getDbName(), table.getTableName(), eventType, rowsCount); + Header header = createHeader(event.getHeader(), + table.getDbName(), + table.getTableName(), + eventType, + rowsCount); RowChange rowChange = rowChangeBuider.build(); if (tableError) { @@ -808,12 +805,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString()); } - private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType) { return createHeader(logHeader, schemaName, tableName, eventType, -1); } - private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType, Integer rowsCount) { + private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType, + Integer rowsCount) { // header会做信息冗余,方便以后做检索或者过滤 Header.Builder headerBuilder = Header.newBuilder(); headerBuilder.setVersion(version); @@ -940,7 +937,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar this.nameBlackFilter = nameBlackFilter; } - public void setTableMetaCache(TableMetaCacheInterface tableMetaCache) { + public void setTableMetaCache(TableMetaCache tableMetaCache) { this.tableMetaCache = tableMetaCache; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index 5c782aa1..4c1cef3c 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -6,7 +6,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaCacheInterface; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; @@ -30,7 +29,7 @@ import com.google.common.cache.LoadingCache; * @author jianghang 2013-1-17 下午10:15:16 * @version 1.0.0 */ -public class TableMetaCache implements TableMetaCacheInterface { +public class TableMetaCache { public static final String COLUMN_NAME = "COLUMN_NAME"; public static final String COLUMN_TYPE = "COLUMN_TYPE"; @@ -100,10 +99,6 @@ public class TableMetaCache implements TableMetaCacheInterface { String createDDL = packet.getFieldValues().get(1); MemoryTableMeta memoryTableMeta = new MemoryTableMeta(); memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null); - String[] strings = table.split("\\."); - if (strings.length > 1) { - table = strings[1]; - } TableMeta tableMeta = memoryTableMeta.find(schema, table); return tableMeta.getFields(); } else { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java deleted file mode 100644 index 0964627d..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/HistoryTableMetaCache.java +++ /dev/null @@ -1,191 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; - -import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; -import com.alibaba.otter.canal.parse.inbound.TableMeta; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.CacheConnectionNull; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception.NoHistoryException; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; - -import java.io.IOException; -import java.util.*; - -public class HistoryTableMetaCache { - private TableMetaStorage tableMetaStorage; - private MysqlConnection metaConnection; - private LoadingCache> cache; // 第一层：数据库名.表名，第二层时间戳，TableMeta - - public HistoryTableMetaCache() { - cache = CacheBuilder.newBuilder().build(new CacheLoader>() { - @Override - public Map load(String tableName) throws Exception { - Long timestamp = new Date().getTime(); - String[] strs = tableName.split("\\."); - String schema = strs[0]; - if (tableMetaStorage != null) { - init(tableMetaStorage.fetchByTableName(tableName)); // 从存储中读取表的历史ddl - } - ResultSetPacket resultSetPacket = connectionQuery("show create table " + tableName); // 获取当前ddl - String currentDdl = resultSetPacket.getFieldValues().get(1); - if (cache.asMap().containsKey(tableName)) { - Map tableMetaMap = cache.getUnchecked(tableName); - if (tableMetaMap.isEmpty()) { - put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema，取时间为当前时间-1s - } else { // 如果table存在历史 - Iterator iterator = tableMetaMap.keySet().iterator(); - Long firstTimestamp = iterator.next(); - TableMeta first = tableMetaMap.get(firstTimestamp); // 拿第一条ddl - if (!first.getDdl().equalsIgnoreCase(currentDdl)) { // 当前ddl与历史第一条不一致，放入当前ddl - put(schema, tableName, currentDdl, calculateNewTimestamp(firstTimestamp)); // 计算放入的timestamp,设为第一条时间+1s - } - } - } else { - put(schema, tableName, currentDdl, timestamp - 1000L); // 放入当前schema - } - return cache.get(tableName); - } - }); - } - - public void init(List entries) throws IOException { - if (entries == null) { - return; - } - for (TableMetaEntry entry : entries) { - try { - put(entry.getSchema(), entry.getTable(), entry.getDdl(), entry.getTimestamp()); - } catch (CacheConnectionNull cacheConnectionNull) { - cacheConnectionNull.printStackTrace(); - } - } - } - - public TableMeta put(String schema, String table, String ddl, Long timestamp) throws CacheConnectionNull, IOException { - ResultSetPacket resultSetPacket; - if (!(ddl.contains("CREATE TABLE") || ddl.contains("create table"))) { // 尝试直接从数据库拉取CREATE TABLE的DDL - resultSetPacket = connectionQuery("show create table " + table); - ddl = resultSetPacket.getFieldValues().get(1); - } else { // CREATE TABLE 的 DDL - resultSetPacket = new ResultSetPacket(); - List fields = new ArrayList(); - String[] strings = table.split("\\."); - String shortTable = table; - if (strings.length > 1) { - shortTable = strings[1]; - } - fields.add(0, shortTable); - fields.add(1, ddl); - resultSetPacket.setFieldValues(fields); - if (metaConnection != null) { - resultSetPacket.setSourceAddress(metaConnection.getAddress()); - } - } - Map tableMetaMap; - if (!cache.asMap().containsKey(table)) { - tableMetaMap = new TreeMap(new Comparator() { - @Override - public int compare(Long o1, Long o2) { - return o2.compareTo(o1); - } - }); - cache.put(table, tableMetaMap); - } else { - tableMetaMap = cache.getUnchecked(table); - } - eliminate(tableMetaMap); // 淘汰旧的TableMeta - TableMeta tableMeta = new TableMeta(schema, table, TableMetaCache.parseTableMeta(schema, table, resultSetPacket)); - if (tableMeta.getDdl() == null) { // 生成的TableMeta有时DDL为null - tableMeta.setDdl(ddl); - } - tableMetaMap.put(timestamp, tableMeta); - return tableMeta; - } - - public TableMeta get(String schema, String table, Long timestamp) throws NoHistoryException, CacheConnectionNull { - Map tableMetaMap = cache.getUnchecked(table); - Iterator iterator = tableMetaMap.keySet().iterator(); - Long selected = null; - while(iterator.hasNext()) { - Long temp = iterator.next(); - if (timestamp > temp) { - selected = temp; - break; - } - } - - if (selected == null) { - iterator = tableMetaMap.keySet().iterator(); - if (iterator.hasNext()) { - selected = iterator.next(); - } else { - throw new NoHistoryException(schema, table); - } - } - - return tableMetaMap.get(selected); - } - - public void clearTableMeta() { - cache.invalidateAll(); - } - - public void clearTableMetaWithSchemaName(String schema) { - for (String tableName : cache.asMap().keySet()) { - String[] strs = tableName.split("\\."); - if (schema.equalsIgnoreCase(strs[0])) { - cache.invalidate(tableName); - } - } - } - - public void clearTableMeta(String schema, String table) { - if (!table.contains(".")) { - table = schema+"."+table; - } - cache.invalidate(table); - } - - // eliminate older table meta in cache - private void eliminate(Map tableMetaMap) { - int MAX_CAPABILITY = 20; - if (tableMetaMap.keySet().size() < MAX_CAPABILITY) { - return; - } - Iterator iterator = tableMetaMap.keySet().iterator(); - while(iterator.hasNext()) { - iterator.next(); - } - iterator.remove(); - } - - private Long calculateNewTimestamp(Long oldTimestamp) { - return oldTimestamp + 1000; - } - - private ResultSetPacket connectionQuery(String query) throws CacheConnectionNull, IOException { - if (metaConnection == null) { - throw new CacheConnectionNull(); - } - try { - return metaConnection.query(query); - } catch (IOException e) { - try { - metaConnection.reconnect(); - return metaConnection.query(query); - } catch (IOException e1) { - throw e1; - } - } - } - - public void setMetaConnection(MysqlConnection metaConnection) { - this.metaConnection = metaConnection; - } - - public void setTableMetaStorage(TableMetaStorage tableMetaStorage) { - this.tableMetaStorage = tableMetaStorage; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java deleted file mode 100644 index 5196ee08..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheInterface.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; - -import com.alibaba.otter.canal.parse.inbound.TableMeta; -import com.alibaba.otter.canal.protocol.position.EntryPosition; - -public interface TableMetaCacheInterface { - - TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position); - - void clearTableMeta(); - - void clearTableMetaWithSchemaName(String schema); - - void clearTableMeta(String schema, String table); - - boolean apply(EntryPosition position, String schema, String ddl, String extra); - - boolean isOnRDS(); - -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java deleted file mode 100644 index eb43158b..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaCacheWithStorage.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; - -import com.alibaba.otter.canal.parse.inbound.TableMeta; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.alibaba.otter.canal.protocol.position.EntryPosition; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; - -public class TableMetaCacheWithStorage implements TableMetaCacheInterface { - - private static Logger logger = LoggerFactory.getLogger(TableMetaCacheWithStorage.class); - private TableMetaStorage tableMetaStorage; // TableMeta存储 - private HistoryTableMetaCache cache = new HistoryTableMetaCache(); // cache - - public TableMetaCacheWithStorage(MysqlConnection con, TableMetaStorage tableMetaStorage) { - this.tableMetaStorage = tableMetaStorage; - InetSocketAddress address = con.getAddress(); - this.tableMetaStorage.setDbAddress(address.getHostName()+":"+address.getPort()); - cache.setMetaConnection(con); - cache.setTableMetaStorage(tableMetaStorage); - if (tableMetaStorage != null) { - try { - cache.init(tableMetaStorage.fetch()); // 初始化，从存储拉取TableMeta - } catch (IOException e) { - logger.error(e.getMessage()); - } - } - } - - @Override - public boolean apply(EntryPosition position, String fullTableName, String ddl, String extra) { - String[] strs = fullTableName.split("\\."); - String schema = strs[0]; - if (schema.equalsIgnoreCase("null")) { // ddl schema为null，放弃处理 - return false; - } - try { - TableMeta tableMeta = cache.get(schema, fullTableName, position.getTimestamp()); - if (!compare(tableMeta, ddl)) { // 获取最近的TableMeta，进行比对 - TableMeta result = cache.put(schema, fullTableName, ddl, calTimestamp(position.getTimestamp())); - if (tableMetaStorage != null && result != null) { // 储存 - tableMetaStorage.store(schema, fullTableName, result.getDdl(), calTimestamp(position.getTimestamp())); - } - } - return true; - } catch (Exception e) { - logger.error(e.toString()); - } - - return false; - } - - @Override - public boolean isOnRDS() { - return false; - } - - /*** - * - * @param schema dbname - * @param table tablename - * @param useCache unused - * @param position timestamp - * @return - */ - @Override - public TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) { - String fulltbName = schema + "." + table; - try { - return cache.get(schema, fulltbName, position.getTimestamp()); - } catch (Exception e) { - logger.error(e.toString()); - } - return null; - } - - @Override - public void clearTableMeta() { - cache.clearTableMeta(); - } - - @Override - public void clearTableMetaWithSchemaName(String schema) { - cache.clearTableMetaWithSchemaName(schema); - } - - @Override - public void clearTableMeta(String schema, String table) { - cache.clearTableMeta(schema, table); - } - - private boolean compare(TableMeta tableMeta, String ddl) { - if (tableMeta == null) { - return false; - } - return tableMeta.getDdl().equalsIgnoreCase(ddl); - } - - private Long calTimestamp(Long timestamp) { - return timestamp; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java deleted file mode 100644 index 03e63ed7..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaEntry.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; - -import java.io.Serializable; - -public class TableMetaEntry implements Serializable { - - private static final long serialVersionUID = -1350200637109107904L; - - private String dbAddress; - private String schema; - private String table; - private String ddl; - private Long timestamp; - - - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public String getDdl() { - return ddl; - } - - public void setDdl(String ddl) { - this.ddl = ddl; - } - - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public String getDbAddress() { - return dbAddress; - } - - public void setDbAddress(String dbAddress) { - this.dbAddress = dbAddress; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java deleted file mode 100644 index ac9d7a0a..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorage.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; - -import java.util.List; - -public interface TableMetaStorage { - - void store(String schema, String table, String ddl, Long timestamp); - - List fetch(); - - List fetchByTableName(String tableName); - - String getDbName(); - - String getDbAddress(); - - void setDbAddress(String address); -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java deleted file mode 100644 index 0e532ebb..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/TableMetaStorageFactory.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta; - -public interface TableMetaStorageFactory { - - TableMetaStorage getTableMetaStorage(); - - String getDbName(); - -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java deleted file mode 100644 index 1dadd78a..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/CacheConnectionNull.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception; - -public class CacheConnectionNull extends Exception{ - - @Override - public String toString() { - return "CacheConnectionNull"; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java deleted file mode 100644 index d8f1d16b..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/exception/NoHistoryException.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.exception; - -public class NoHistoryException extends Exception{ - - private String dbName; - private String tbName; - - public NoHistoryException(String dbName, String tbName) { - this.dbName = dbName; - this.tbName = tbName; - } - - public void printTableName() { - System.out.println(dbName+"."+tbName); - } - - @Override - public String toString() { - return "NioHistoryException: " + dbName + " " + tbName; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java deleted file mode 100644 index 661986a1..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaCallback.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql; - -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry; - -import java.util.List; - -public interface MySqlTableMetaCallback { - - void save(String dbAddress, String schema, String table,String ddl, Long timestamp); - - List fetch(String dbAddress, String dbName); - - List fetch(String dbAddress, String dbName, String tableName); -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java deleted file mode 100644 index f120ef80..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorage.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql; - -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaEntry; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; - -import java.util.List; - -public class MySqlTableMetaStorage implements TableMetaStorage { - private MySqlTableMetaCallback mySqlTableMetaCallback; - private String dbName; - private String dbAddress; - - MySqlTableMetaStorage(MySqlTableMetaCallback callback, String dbName) { - mySqlTableMetaCallback = callback; - this.dbName = dbName; - } - - - @Override - public void store(String schema, String table, String ddl, Long timestamp) { - mySqlTableMetaCallback.save(dbAddress, schema, table, ddl, timestamp); - } - - @Override - public List fetch() { - return mySqlTableMetaCallback.fetch(dbAddress, dbName); - } - - @Override - public List fetchByTableName(String tableName) { - return mySqlTableMetaCallback.fetch(dbAddress, dbName, tableName); - } - - @Override - public String getDbName() { - return dbName; - } - - @Override - public String getDbAddress() { - return dbAddress; - } - - @Override - public void setDbAddress(String address) { - this.dbAddress = address; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java deleted file mode 100644 index 0eee235d..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/impl/mysql/MySqlTableMetaStorageFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql; - -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorage; -import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.TableMetaStorageFactory; - -public class MySqlTableMetaStorageFactory implements TableMetaStorageFactory { - - private MySqlTableMetaCallback mySQLTableMetaCallback; - private String dbName; - - public MySqlTableMetaStorageFactory(MySqlTableMetaCallback callback, String dbName) { - mySQLTableMetaCallback = callback; - this.dbName = dbName; - } - - @Override - public TableMetaStorage getTableMetaStorage() { - return new MySqlTableMetaStorage(mySQLTableMetaCallback, dbName); - } - - @Override - public String getDbName() { - return dbName; - } - -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DefaultTableMetaTSDBFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DefaultTableMetaTSDBFactory.java new file mode 100644 index 00000000..00e2744b --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DefaultTableMetaTSDBFactory.java @@ -0,0 +1,19 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +/** + * @author agapple 2017年10月11日 下午8:45:40 + * @since 1.0.25 + */ +public class DefaultTableMetaTSDBFactory implements TableMetaTSDBFactory { + + /** + * 代理一下tableMetaTSDB的获取,使用隔离的spring定义 + */ + public TableMetaTSDB build(String destination, String springXml) { + return TableMetaTSDBBuilder.build(destination, springXml); + } + + public void destory(String destination) { + TableMetaTSDBBuilder.destory(destination); + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java index 8e37ef71..107e1013 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java @@ -10,12 +10,15 @@ import org.springframework.context.support.ClassPathXmlApplicationContext; import com.google.common.collect.Maps; /** - * @author agapple 2017年10月11日 下午8:45:40 + * tableMeta构造器 + * + * @author agapple 2018年8月8日 上午11:01:08 * @since 1.0.25 */ + public class TableMetaTSDBBuilder { - protected final static Logger logger = LoggerFactory.getLogger(TableMetaTSDBBuilder.class); + protected final static Logger logger = LoggerFactory.getLogger(DefaultTableMetaTSDBFactory.class); private static ConcurrentMap contexts = Maps.newConcurrentMap(); /** diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBFactory.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBFactory.java new file mode 100644 index 00000000..950645a8 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBFactory.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +/** + * tableMeta构造器,允许重载实现 + * + * @author agapple 2018年8月8日 上午11:01:08 + * @since 1.0.26 + */ + +public interface TableMetaTSDBFactory { + + /** + * 代理一下tableMetaTSDB的获取,使用隔离的spring定义 + */ + public TableMetaTSDB build(String destination, String springXml); + + public void destory(String destination); +}