From ae0f43d1c71a8e3591adf27a7fc3b1ce60587d70 Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Wed, 1 Aug 2018 08:17:55 +0800 Subject: [PATCH 01/21] netty3 future listener for metrics. --- .../listener/ChannelFutureAggregator.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java new file mode 100644 index 00000000..aa9b0da3 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java @@ -0,0 +1,42 @@ +package com.alibaba.otter.canal.server.netty.listener; + +import com.alibaba.otter.canal.protocol.CanalPacket; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelFutureListener; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author Chuanyi Li + */ +public class ChannelFutureAggregator implements ChannelFutureListener { + + private String dest; + + private CanalPacket.PacketType type; + + private int amount; + + private int errorCode; + + public ChannelFutureAggregator(String dest, CanalPacket.PacketType type, int amount) { + this(dest, type, amount, 0); + } + + public ChannelFutureAggregator(String dest, CanalPacket.PacketType type, int amount, int errorCode) { + this.dest = dest; + this.type = type; + this.amount = amount; + this.errorCode = errorCode; + } + + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (future.isSuccess()) { + ConcurrentHashMap map = new ConcurrentHashMap(); + map.putIfAbsent("", ""); + } else { + Throwable t = future.getCause(); + } + } +} From 93ebc7b973b282c38d9cfacd7d7846cac309403b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Wed, 1 Aug 2018 18:25:49 +0800 Subject: [PATCH 02/21] fixed unused repo --- pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index 59c5bc01..9a14868b 100644 --- a/pom.xml +++ b/pom.xml @@ -66,16 +66,6 @@ false - - alibaba - http://code.alibabatech.com/mvn/releases/ - - true - - - false - - sonatype sonatype From 26edbeaa280b8c3617fea51f16f9358d67f659cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Thu, 2 Aug 2018 13:58:41 +0800 Subject: [PATCH 03/21] fixed kafka performance --- deployer/src/main/resources/canal.properties | 1 + .../resources/spring/default-instance.xml | 1 + .../main/resources/spring/file-instance.xml | 1 + .../main/resources/spring/group-instance.xml | 1 + .../main/resources/spring/local-instance.xml | 1 + .../main/resources/spring/memory-instance.xml | 1 + .../kafka/client/MessageDeserializer.java | 4 +- .../client/running/ClientRunningData.java | 4 +- .../client/running/ClientRunningMonitor.java | 82 ++++++++++--------- .../otter/canal/kafka/CanalServerStarter.java | 6 +- .../kafka/producer/CanalKafkaProducer.java | 51 ++++++------ .../kafka/producer/CanalKafkaStarter.java | 5 +- .../canal/kafka/producer/KafkaProperties.java | 25 ++++-- .../kafka/producer/MessageSerializer.java | 59 ++++++++++--- kafka/src/main/resources/kafka.yml | 1 + 15 files changed, 154 insertions(+), 89 deletions(-) diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 23ec03a1..0a11a5c9 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -43,6 +43,7 @@ canal.instance.filter.query.dml = false canal.instance.filter.query.ddl = false canal.instance.filter.table.error = false canal.instance.filter.rows = false +canal.instance.filter.transaction.entry = false # binlog format/image check canal.instance.binlog.format = ROW,STATEMENT,MIXED diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index c4465df1..9bf28896 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -81,6 +81,7 @@ + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 340a8158..6d263717 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -67,6 +67,7 @@ + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 60d069dc..60ceff01 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -64,6 +64,7 @@ + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index 0b2f96aa..2020d0d6 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -67,6 +67,7 @@ + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index 8fec8918..c08f3f12 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -64,6 +64,7 @@ + diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java index 1cb3daba..70f2fc06 100644 --- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java +++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/MessageDeserializer.java @@ -29,7 +29,9 @@ public class MessageDeserializer implements Deserializer { @Override public Message deserialize(String topic, byte[] data) { try { - if (data == null) return null; + if (data == null) { + return null; + } else { CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data); switch (p.getType()) { diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java index 3eab85ed..7bbbabd7 100644 --- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java +++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningData.java @@ -8,8 +8,8 @@ package com.alibaba.otter.canal.kafka.client.running; */ public class ClientRunningData { - private String groupId; - private String address; + private String groupId; + private String address; private boolean active = true; public String getGroupId() { diff --git a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java index 3bdbfe6e..0f61211e 100644 --- a/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java +++ b/kafka-client/src/main/java/com/alibaba/otter/canal/kafka/client/running/ClientRunningMonitor.java @@ -1,12 +1,11 @@ package com.alibaba.otter.canal.kafka.client.running; -import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; -import com.alibaba.otter.canal.common.utils.AddressUtils; -import com.alibaba.otter.canal.common.utils.BooleanMutex; -import com.alibaba.otter.canal.common.utils.JsonUtils; -import com.alibaba.otter.canal.common.zookeeper.ZkClientx; -import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; -import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import java.text.MessageFormat; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + import org.I0Itec.zkclient.IZkDataListener; import org.I0Itec.zkclient.exception.ZkException; import org.I0Itec.zkclient.exception.ZkInterruptedException; @@ -17,11 +16,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import java.text.MessageFormat; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.utils.BooleanMutex; +import com.alibaba.otter.canal.common.utils.JsonUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + /** * kafka client running状态信息 @@ -31,13 +33,18 @@ import java.util.concurrent.TimeUnit; */ public class ClientRunningMonitor extends AbstractCanalLifeCycle { - private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics"; + private static final String TOPIC_ROOT_NODE = ZookeeperPathUtils.CANAL_ROOT_NODE + + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics"; - private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "{0}"; + private static final String TOPIC_NODE = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + + "{0}"; - private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "{1}"; + private static final String TOPIC_CLIENTID_NODE = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + + "{1}"; - private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + ZookeeperPathUtils.RUNNING_NODE; + private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE + + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + + ZookeeperPathUtils.RUNNING_NODE; private static String getTopicClientRunning(String topic, String groupId) { return MessageFormat.format(TOPIC_CLIENTID_RUNNING_NODE, topic, groupId); @@ -47,21 +54,21 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { return MessageFormat.format(TOPIC_CLIENTID_NODE, topic, groupId); } - private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class); - private ZkClientx zkClient; - private String topic; - private ClientRunningData clientData; - private IZkDataListener dataListener; - private BooleanMutex mutex = new BooleanMutex(false); - private volatile boolean release = false; + private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class); + private ZkClientx zkClient; + private String topic; + private ClientRunningData clientData; + private IZkDataListener dataListener; + private BooleanMutex mutex = new BooleanMutex(false); + private volatile boolean release = false; private volatile ClientRunningData activeData; - private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); - private ClientRunningListener listener; - private int delayTime = 5; + private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); + private ClientRunningListener listener; + private int delayTime = 5; - private static Integer virtualPort; + private static Integer virtualPort; - public ClientRunningMonitor() { + public ClientRunningMonitor(){ if (virtualPort == null) { Random rand = new Random(); virtualPort = rand.nextInt(9000) + 1000; @@ -108,7 +115,6 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { } - public void start() { super.start(); @@ -123,7 +129,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { String path = getTopicClientRunning(this.topic, clientData.getGroupId()); zkClient.unsubscribeDataChanges(path, dataListener); releaseRunning(); // 尝试一下release - //Fix issue #697 + // Fix issue #697 if (delayExector != null) { delayExector.shutdown(); } @@ -159,13 +165,12 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { } } } catch (ZkNoNodeException e) { - zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), - true); // 尝试创建父节点 + zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), true); // 尝试创建父节点 initRunning(); } catch (Throwable t) { logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].", - topic), - t); + topic), + t); // 出现任何异常尝试release releaseRunning(); throw new CanalClientException("something goes wrong in initRunning method. ", t); @@ -187,7 +192,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { */ public boolean check() { String path = getTopicClientRunning(this.topic, clientData.getGroupId()); - //ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + // ZookeeperPathUtils.getDestinationClientRunning(this.destination, + // clientData.getClientId()); try { byte[] bytes = zkClient.readData(path); ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class); @@ -196,8 +202,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { boolean result = isMine(activeData.getAddress()); if (!result) { logger.warn("canal is running in [{}] , but not in [{}]", - activeData.getAddress(), - clientData.getAddress()); + activeData.getAddress(), + clientData.getAddress()); } return result; } catch (ZkNoNodeException e) { @@ -235,7 +241,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle { if (listener != null) { // 触发回调 listener.processActiveEnter(); - this.clientData.setAddress(/*address*/AddressUtils.getHostIp() + ":" + virtualPort); + this.clientData.setAddress(/* address */AddressUtils.getHostIp() + ":" + virtualPort); String path = getTopicClientRunning(this.topic, clientData.getGroupId()); // 序列化 diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java index e0b2d1d8..573f0eac 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/CanalServerStarter.java @@ -17,9 +17,9 @@ import com.alibaba.otter.canal.deployer.CanalController; */ public class CanalServerStarter { - private static final String CLASSPATH_URL_PREFIX = "classpath:"; - private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class); - private volatile static boolean running = false; + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(CanalServerStarter.class); + private volatile static boolean running = false; public static void init() { try { diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java index 3318a89e..4543aac1 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java @@ -3,7 +3,6 @@ package com.alibaba.otter.canal.kafka.producer; import java.io.IOException; import java.util.Properties; -import com.google.protobuf.ByteString; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -12,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.kafka.producer.KafkaProperties.Topic; -import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.Message; /** @@ -52,31 +50,32 @@ public class CanalKafkaProducer { } public void send(Topic topic, Message message) throws IOException { - boolean valid = false; - if (message != null) { - if (message.isRaw() && !message.getRawEntries().isEmpty()) { - for (ByteString byteString : message.getRawEntries()) { - CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); - if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN - && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { - valid = true; - break; - } - } - } else if (!message.getEntries().isEmpty()){ - for (CanalEntry.Entry entry : message.getEntries()) { - if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN - && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { - valid = true; - break; - } - } - } - } - if (!valid) { - return; - } + // set canal.instance.filter.transaction.entry = true + // boolean valid = false; + // if (message != null) { + // if (message.isRaw() && !message.getRawEntries().isEmpty()) { + // for (ByteString byteString : message.getRawEntries()) { + // CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString); + // if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN + // && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { + // valid = true; + // break; + // } + // } + // } else if (!message.getEntries().isEmpty()){ + // for (CanalEntry.Entry entry : message.getEntries()) { + // if (entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONBEGIN + // && entry.getEntryType() != CanalEntry.EntryType.TRANSACTIONEND) { + // valid = true; + // break; + // } + // } + // } + // } + // if (!valid) { + // return; + // } ProducerRecord record; if (topic.getPartition() != null) { record = new ProducerRecord(topic.getTopic(), topic.getPartition(), null, message); diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java index 0ff018c2..e88f9354 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java @@ -52,7 +52,10 @@ public class CanalKafkaStarter { // 初始化 kafka producer canalKafkaProducer = new CanalKafkaProducer(); canalKafkaProducer.init(kafkaProperties); - + // set filterTransactionEntry + if (kafkaProperties.isFilterTransactionEntry()) { + System.setProperty("canal.instance.filter.transaction.entry", "true"); + } // 对应每个instance启动一个worker线程 List destinations = kafkaProperties.getCanalDestinations(); diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java index 0b346137..8e193aac 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/KafkaProperties.java @@ -13,15 +13,15 @@ import java.util.Set; */ public class KafkaProperties { - private String servers = "localhost:6667"; - private int retries = 0; - private int batchSize = 16384; - private int lingerMs = 1; - private long bufferMemory = 33554432L; + private String servers = "localhost:6667"; + private int retries = 0; + private int batchSize = 16384; + private int lingerMs = 1; + private long bufferMemory = 33554432L; + private boolean filterTransactionEntry = true; + private int canalBatchSize = 5; - private int canalBatchSize = 5; - - private List canalDestinations = new ArrayList(); + private List canalDestinations = new ArrayList(); public static class CanalDestination { @@ -158,4 +158,13 @@ public class KafkaProperties { public void setCanalDestinations(List canalDestinations) { this.canalDestinations = canalDestinations; } + + public boolean isFilterTransactionEntry() { + return filterTransactionEntry; + } + + public void setFilterTransactionEntry(boolean filterTransactionEntry) { + this.filterTransactionEntry = filterTransactionEntry; + } + } diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java index 17fca117..02336004 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/MessageSerializer.java @@ -1,13 +1,19 @@ package com.alibaba.otter.canal.kafka.producer; -import com.alibaba.otter.canal.protocol.CanalEntry; -import com.alibaba.otter.canal.protocol.CanalPacket; -import com.alibaba.otter.canal.protocol.Message; +import java.util.List; +import java.util.Map; + import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serializer; import org.springframework.util.CollectionUtils; -import java.util.Map; +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.CanalPacket; +import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; +import com.alibaba.otter.canal.protocol.Message; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.WireFormat; /** * Kafka Message类的序列化 @@ -25,20 +31,53 @@ public class MessageSerializer implements Serializer { public byte[] serialize(String topic, Message data) { try { if (data != null) { - CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); if (data.getId() != -1) { if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) { - messageBuilder.addAllMessages(data.getRawEntries()); + // for performance + List rowEntries = data.getRawEntries(); + // message size + int messageSize = 0; + messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, data.getId()); + + int dataSize = 0; + for (int i = 0; i < rowEntries.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i)); + } + messageSize += dataSize; + messageSize += 1 * rowEntries.size(); + // packet size + int size = 0; + size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, + PacketType.MESSAGES.getNumber()); + size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + + messageSize; + // build data + byte[] body = new byte[size]; + CodedOutputStream output = CodedOutputStream.newInstance(body); + output.writeEnum(3, PacketType.MESSAGES.getNumber()); + + output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(messageSize); + // message + output.writeInt64(1, data.getId()); + for (int i = 0; i < rowEntries.size(); i++) { + output.writeBytes(2, rowEntries.get(i)); + } + output.checkNoSpaceLeft(); + return body; } else if (!CollectionUtils.isEmpty(data.getEntries())) { + CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder(); for (CanalEntry.Entry entry : data.getEntries()) { messageBuilder.addMessages(entry.toByteString()); } + + CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); + packetBuilder.setType(CanalPacket.PacketType.MESSAGES); + packetBuilder.setBody(messageBuilder.build().toByteString()); + return packetBuilder.build().toByteArray(); } } - CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder(); - packetBuilder.setType(CanalPacket.PacketType.MESSAGES); - packetBuilder.setBody(messageBuilder.build().toByteString()); - return packetBuilder.build().toByteArray(); } } catch (Exception e) { throw new SerializationException("Error when serializing message to byte[] "); diff --git a/kafka/src/main/resources/kafka.yml b/kafka/src/main/resources/kafka.yml index ab214319..d6ccc51a 100644 --- a/kafka/src/main/resources/kafka.yml +++ b/kafka/src/main/resources/kafka.yml @@ -5,6 +5,7 @@ lingerMs: 1 bufferMemory: 33554432 # canal的批次大小，单位 k canalBatchSize: 50 +filterTransactionEntry: true canalDestinations: - canalDestination: example From 368554166c2305bd4bf7381980e741273820ffe0 Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Thu, 2 Aug 2018 17:36:22 +0800 Subject: [PATCH 04/21] Builder of Aggregator. --- .../listener/ChannelFutureAggregator.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java index aa9b0da3..e3f60dca 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java @@ -4,6 +4,7 @@ import com.alibaba.otter.canal.protocol.CanalPacket; import org.jboss.netty.channel.ChannelFuture; import org.jboss.netty.channel.ChannelFutureListener; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; /** @@ -39,4 +40,64 @@ public class ChannelFutureAggregator implements ChannelFutureListener { Throwable t = future.getCause(); } } + + public static class ClientRequestResult { + + private String dest; + + private CanalPacket.PacketType type; + + private int amount; + + private int errorCode; + + private ClientRequestResult() {} + + private ClientRequestResult(Builder builder) { + this.dest = builder.dest; + this.type = builder.type; + this.amount = builder.amount; + this.errorCode = builder.errorCode; + } + + private static class Builder { + + private String dest; + private CanalPacket.PacketType type; + private int amount; + private int errorCode; + + public Builder dest(String dest) { + this.dest = dest; + return this; + } + + public Builder type(CanalPacket.PacketType type) { + this.type = type; + return this; + } + + public Builder amount(int amount) { + this.amount = amount; + return this; + } + + public Builder errorCode(int errorCode) { + this.errorCode = errorCode; + return this; + } + + public Builder fromPrototype(ClientRequestResult prototype) { + dest = prototype.dest; + type = prototype.type; + amount = prototype.amount; + errorCode = prototype.errorCode; + return this; + } + + public ClientRequestResult build() { + return new ClientRequestResult(this); + } + } + } } From da1ce51a0ff52ee289ce96243bc4c5952b2dd7b6 Mon Sep 17 00:00:00 2001 From: winger Date: Fri, 3 Aug 2018 11:11:38 +0800 Subject: [PATCH 05/21] =?UTF-8?q?=E6=94=AF=E6=8C=81GTID=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E9=80=9A=E8=BF=87=E8=8E=B7=E5=8F=96=E5=BD=93?= =?UTF-8?q?=E5=89=8Dgtid=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/parse/inbound/mysql/MysqlEventParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 6c2b9d4d..e640364f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -643,6 +643,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation"); } EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1))); + if (isGTIDMode && fields.size() > 4) { + endPosition.setGtid(fields.get(4)); + } return endPosition; } catch (IOException e) { throw new CanalParseException("command : 'show master status' has an error!", e); From c7fc149e7ece2572b2597166190fe164f8604466 Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Fri, 3 Aug 2018 17:32:48 +0800 Subject: [PATCH 06/21] Netty server/client profiler skeleton. --- pom.xml | 2 +- .../netty/CanalServerWithNettyProfiler.java | 92 +++++++++++++ .../server/netty/ClientInstanceProfiler.java | 14 ++ .../netty/ClientInstanceProfilerFactory.java | 10 ++ .../server/netty/handler/SessionHandler.java | 1 + .../listener/ChannelFutureAggregator.java | 124 +++++++++++++----- 6 files changed, 207 insertions(+), 36 deletions(-) create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java diff --git a/pom.xml b/pom.xml index 9a14868b..46d103e3 100644 --- a/pom.xml +++ b/pom.xml @@ -247,7 +247,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_520 + 2.0.0_preview_186 com.alibaba diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java new file mode 100644 index 00000000..5b629909 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java @@ -0,0 +1,92 @@ +package com.alibaba.otter.canal.server.netty; + +import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator.ClientRequestResult; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Chuanyi Li + */ +public class CanalServerWithNettyProfiler { + + private static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); + + private volatile ClientInstanceProfilerFactory factory; + + private final ConcurrentMap cliPfs; + + private final CanalServerWithEmbedded server; + + private static class SingletonHolder { + private static CanalServerWithNettyProfiler SINGLETON = new CanalServerWithNettyProfiler(); + } + + private CanalServerWithNettyProfiler() { + this.factory = DISABLED; + this.cliPfs = new ConcurrentHashMap(); + this.server = CanalServerWithEmbedded.instance(); + } + + public static CanalServerWithNettyProfiler profiler() { + return SingletonHolder.SINGLETON; + } + + public void profiling(String dest, ClientRequestResult result) { + if (isDisabled()) { + return; + } + ClientInstanceProfiler profiler = tryGet(dest); + if (profiler != null) { + profiler.profiling(result); + } + } + + /** + * Remove instance profiler for specified instance. + * Only accepted while instance is not running. + * @param dest canal instance destination + */ + public void remove(String dest) throws IllegalStateException { + if (isDisabled()) { + return; + } + synchronized (cliPfs) { + if (server.isStart(dest)) { + throw new IllegalStateException("Instance profiler should not be removed while running."); + } + cliPfs.remove(dest); + } + } + + public void setFactory(ClientInstanceProfilerFactory factory) { + this.factory = factory; + } + + private boolean isDisabled() { + return factory == DISABLED || factory == null; + } + + private ClientInstanceProfiler tryGet(String dest) { + //try fast get + ClientInstanceProfiler profiler = cliPfs.get(dest); + if (profiler == null) { + synchronized (cliPfs) { + if (server.isStart(dest)) { + // avoid overwriting + cliPfs.putIfAbsent(dest, factory.create()); + profiler = cliPfs.get(dest); + } + } + } + return profiler; + } + + private static class DefaultClientInstanceProfilerFactory implements ClientInstanceProfilerFactory { + @Override + public ClientInstanceProfiler create() { + throw new UnsupportedOperationException(); + } + } + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java new file mode 100644 index 00000000..6c9ffc8b --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java @@ -0,0 +1,14 @@ +package com.alibaba.otter.canal.server.netty; + +import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator; + +/** + * @author Chuanyi Li + */ +public interface ClientInstanceProfiler { + + String getDestination(); + + void profiling(ChannelFutureAggregator.ClientRequestResult result); + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java new file mode 100644 index 00000000..30075ba1 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java @@ -0,0 +1,10 @@ +package com.alibaba.otter.canal.server.netty; + +/** + * @author Chuanyi Li + */ +public interface ClientInstanceProfilerFactory { + + ClientInstanceProfiler create(); + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java index 11bba043..7ea861d4 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java @@ -51,6 +51,7 @@ public class SessionHandler extends SimpleChannelHandler { public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { logger.info("message receives in session handler..."); + long start = System.currentTimeMillis(); ChannelBuffer buffer = (ChannelBuffer) e.getMessage(); Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array()); ClientIdentity clientIdentity = null; diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java index e3f60dca..37ee88e2 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java @@ -1,103 +1,157 @@ package com.alibaba.otter.canal.server.netty.listener; import com.alibaba.otter.canal.protocol.CanalPacket; +import com.google.common.base.Preconditions; +import com.google.protobuf.GeneratedMessage; import org.jboss.netty.channel.ChannelFuture; import org.jboss.netty.channel.ChannelFutureListener; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; +import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.profiler; /** * @author Chuanyi Li */ public class ChannelFutureAggregator implements ChannelFutureListener { - private String dest; + private ClientRequestResult result; - private CanalPacket.PacketType type; - - private int amount; - - private int errorCode; - - public ChannelFutureAggregator(String dest, CanalPacket.PacketType type, int amount) { - this(dest, type, amount, 0); + public ChannelFutureAggregator(String dest, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency) { + this(dest, request, type, amount, latency, 0); } - public ChannelFutureAggregator(String dest, CanalPacket.PacketType type, int amount, int errorCode) { - this.dest = dest; - this.type = type; - this.amount = amount; - this.errorCode = errorCode; + private ChannelFutureAggregator(String dest, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, int errorCode) { + this.result = new ClientRequestResult.Builder() + .dest(dest) + .type(type) + .request(request) + .amount(amount) + .latency(latency) + .errorCode(errorCode) + .build(); } @Override - public void operationComplete(ChannelFuture future) throws Exception { - if (future.isSuccess()) { - ConcurrentHashMap map = new ConcurrentHashMap(); - map.putIfAbsent("", ""); - } else { - Throwable t = future.getCause(); + public void operationComplete(ChannelFuture future) { + // profiling after I/O operation + if (future.getCause() != null) { + result.channelError = future.getCause(); } + profiler().profiling(result.dest, result); } + /** + * Client request result pojo + */ public static class ClientRequestResult { private String dest; - private CanalPacket.PacketType type; - - private int amount; - - private int errorCode; + private GeneratedMessage request; + private int amount; + private long latency; + private int errorCode; + private Throwable channelError; private ClientRequestResult() {} private ClientRequestResult(Builder builder) { - this.dest = builder.dest; - this.type = builder.type; + this.dest = Preconditions.checkNotNull(builder.dest); + this.type = Preconditions.checkNotNull(builder.type); + this.request = Preconditions.checkNotNull(builder.request); this.amount = builder.amount; + this.latency = builder.latency; this.errorCode = builder.errorCode; + this.channelError = builder.channelError; } - private static class Builder { + // auto-generated + public static class Builder { private String dest; private CanalPacket.PacketType type; + private GeneratedMessage request; private int amount; + private long latency; private int errorCode; + private Throwable channelError; - public Builder dest(String dest) { + Builder dest(String dest) { this.dest = dest; return this; } - public Builder type(CanalPacket.PacketType type) { + Builder type(CanalPacket.PacketType type) { this.type = type; return this; } - public Builder amount(int amount) { + Builder request(GeneratedMessage request) { + this.request = request; + return this; + } + + Builder amount(int amount) { this.amount = amount; return this; } - public Builder errorCode(int errorCode) { + Builder latency(long latency) { + this.latency = latency; + return this; + } + + Builder errorCode(int errorCode) { this.errorCode = errorCode; return this; } + public Builder channelError(Throwable channelError) { + this.channelError = channelError; + return this; + } + public Builder fromPrototype(ClientRequestResult prototype) { dest = prototype.dest; type = prototype.type; + request = prototype.request; amount = prototype.amount; + latency = prototype.latency; errorCode = prototype.errorCode; + channelError = prototype.channelError; return this; } - public ClientRequestResult build() { + ClientRequestResult build() { return new ClientRequestResult(this); } } + // getters + public String getDest() { + return dest; + } + + public CanalPacket.PacketType getType() { + return type; + } + + public GeneratedMessage getRequest() { + return request; + } + + public int getAmount() { + return amount; + } + + public long getLatency() { + return latency; + } + + public int getErrorCode() { + return errorCode; + } + + public Throwable getChannelError() { + return channelError; + } } } From b7431eb7d0fac415787bc7c7b7ebb4d4f0d62bb8 Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Fri, 3 Aug 2018 17:42:57 +0800 Subject: [PATCH 07/21] Reformat code. --- .../canal/server/netty/CanalServerWithNettyProfiler.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java index 5b629909..9e9d9a39 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java @@ -10,12 +10,9 @@ import java.util.concurrent.ConcurrentMap; */ public class CanalServerWithNettyProfiler { - private static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); - + private static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); private volatile ClientInstanceProfilerFactory factory; - private final ConcurrentMap cliPfs; - private final CanalServerWithEmbedded server; private static class SingletonHolder { From 7fd958a490d9a5bb1d42d43c84ddbc5e6bf4d6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 3 Aug 2018 17:48:32 +0800 Subject: [PATCH 08/21] fixed issue #801 , support canal docker --- .gitignore | 2 + .../otter/canal/deployer/CanalController.java | 13 +- deployer/src/main/resources/canal.properties | 9 +- .../resources/example/instance.properties | 19 ++- .../resources/spring/default-instance.xml | 4 +- .../main/resources/spring/file-instance.xml | 4 +- .../main/resources/spring/group-instance.xml | 8 +- .../main/resources/spring/local-instance.xml | 2 +- .../main/resources/spring/memory-instance.xml | 4 +- docker/Dockerfile | 69 +++++++++++ docker/build.sh | 30 +++++ docker/image/admin/app.sh | 117 ++++++++++++++++++ docker/image/admin/bin/clean_log | 2 + docker/image/admin/bin/clean_log.sh | 45 +++++++ docker/image/admin/health.sh | 13 ++ docker/image/alidata/bin/exec_rc_local.sh | 11 ++ docker/image/alidata/bin/lark-wait | 6 + docker/image/alidata/bin/main.sh | 27 ++++ docker/image/alidata/init/02init-sshd.sh | 19 +++ docker/image/alidata/init/fix-hosts.py | 66 ++++++++++ docker/image/alidata/lib/proc.sh | 40 ++++++ docker/run.sh | 92 ++++++++++++++ .../parse/driver/mysql/MysqlConnector.java | 2 +- 23 files changed, 580 insertions(+), 24 deletions(-) create mode 100644 docker/Dockerfile create mode 100644 docker/build.sh create mode 100755 docker/image/admin/app.sh create mode 100644 docker/image/admin/bin/clean_log create mode 100755 docker/image/admin/bin/clean_log.sh create mode 100755 docker/image/admin/health.sh create mode 100755 docker/image/alidata/bin/exec_rc_local.sh create mode 100755 docker/image/alidata/bin/lark-wait create mode 100755 docker/image/alidata/bin/main.sh create mode 100755 docker/image/alidata/init/02init-sshd.sh create mode 100755 docker/image/alidata/init/fix-hosts.py create mode 100644 docker/image/alidata/lib/proc.sh create mode 100644 docker/run.sh diff --git a/.gitignore b/.gitignore index 3f1a5d6b..90bd720f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ jtester.properties .idea/ *.iml .DS_Store +*.tar.gz +*.rpm diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java index 88f6e9a3..0a2c91f8 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java @@ -379,7 +379,18 @@ public class CanalController { } private String getProperty(Properties properties, String key) { - return StringUtils.trim(properties.getProperty(StringUtils.trim(key))); + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); } public void start() throws Throwable { diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 0a11a5c9..2bff29ae 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -3,7 +3,7 @@ ################################################# canal.id= 1 canal.ip= -canal.port= 11111 +canal.port=11111 canal.zkServers= # flush data to zk canal.zookeeper.flush.period = 1000 @@ -59,6 +59,13 @@ canal.instance.parser.parallel = true ## disruptor ringbuffer size, must be power of 2 canal.instance.parser.parallelBufferSize = 256 +# table meta tsdb info +canal.instance.tsdb.enable=true +canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +canal.instance.tsdb.dbUsername=canal +canal.instance.tsdb.dbPassword=canal + ################################################# ######### destinations ############# ################################################# diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index a3703b86..e0cf0064 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -1,11 +1,12 @@ ################################################# -## mysql serverId -canal.instance.mysql.slaveId=0 +## mysql serverId , v1.0.26+ will autoGen +# canal.instance.mysql.slaveId=0 + +# enable gtid use true/false +canal.instance.gtidon=false # position info canal.instance.master.address=127.0.0.1:3306 -# enable gtid use true/false -canal.instance.gtidon=false canal.instance.master.journal.name= canal.instance.master.position= canal.instance.master.timestamp= @@ -13,23 +14,21 @@ canal.instance.master.gtid= # table meta tsdb info canal.instance.tsdb.enable=true -canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} -canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb -canal.instance.tsdb.dbUsername=canal -canal.instance.tsdb.dbPassword=canal - +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal #canal.instance.standby.address = #canal.instance.standby.journal.name = #canal.instance.standby.position = #canal.instance.standby.timestamp = #canal.instance.standby.gtid= + # username/password canal.instance.dbUsername=canal canal.instance.dbPassword=canal -canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 + # table regex canal.instance.filter.regex=.*\\..* # table black regex diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index 9bf28896..6b455c83 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -149,7 +149,7 @@ - + @@ -157,7 +157,7 @@ - + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 6d263717..377428a9 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -134,7 +134,7 @@ - + @@ -142,7 +142,7 @@ - + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 60ceff01..44f1ab3b 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -131,7 +131,7 @@ - + @@ -139,7 +139,7 @@ - + @@ -229,7 +229,7 @@ - + @@ -237,7 +237,7 @@ - + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index 2020d0d6..930ecb7b 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -114,7 +114,7 @@ - + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index c08f3f12..cd387b8f 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -122,7 +122,7 @@ - + @@ -130,7 +130,7 @@ - + diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..7e10085b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,69 @@ +FROM centos:centos6.7 + +MAINTAINER agapple (jianghang115@gmail.com) + +# install system +RUN \ + /bin/cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo 'root:Hello1234' | chpasswd && \ + groupadd -r admin && useradd -g admin admin && \ + yum install -y man && \ + yum install -y dstat && \ + yum install -y unzip && \ + yum install -y nc && \ + yum install -y openssh-server && \ + yum install -y tar && \ + yum install -y which && \ + yum install -y wget && \ + yum install -y perl && \ + yum install -y file && \ + ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \ + ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \ + sed -ri 's/session required pam_loginuid.so/#session required pam_loginuid.so/g' /etc/pam.d/sshd && \ + sed -i -e 's/^#Port 22$/Port 2222/' /etc/ssh/sshd_config && \ + mkdir -p /root/.ssh && chown root.root /root && chmod 700 /root/.ssh && \ + yum install -y cronie && \ + sed -i '/session required pam_loginuid.so/d' /etc/pam.d/crond && \ + yum clean all && \ + true + +# install canal +COPY image/ /tmp/docker/ +COPY canal.deployer-*.tar.gz /home/admin/ +COPY jdk-8-linux-x64.rpm /tmp/ + +RUN \ + cp -R /tmp/docker/alidata /alidata && \ + chmod +x /alidata/bin/* && \ + mkdir -p /home/admin && \ + cp -R /tmp/docker/admin/* /home/admin/ && \ + /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \ + + touch /var/lib/rpm/* && \ + yum -y install /tmp/jdk-8-linux-x64.rpm && \ + /bin/rm -f /tmp/jdk-8-linux-x64.rpm && \ + + echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ + echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ + /bin/mv /home/admin/bin/canal-check /etc/cron.d && \ + /bin/mv /home/admin/bin/clean_log /etc/cron.d && \ + chmod +x /home/admin/bin/canal-process-check.py && \ + + mkdir -p /home/admin/canal-server && \ + tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ + /bin/rm -f /home/admin/canal.deployer-*.tar.gz && \ + + mkdir -p home/admin/canal-server/logs && \ + chmod +x /home/admin/*.sh && \ + chmod +x /home/admin/bin/*.sh && \ + chown admin: -R /home/admin && \ + yum clean all && \ + true + +# 2222 sys , 8080 web , 8000 debug , 11111 canal +EXPOSE 2222 11111 8000 8080 + +WORKDIR /home/admin + +ENTRYPOINT [ "/alidata/bin/main.sh" ] +CMD [ "/home/admin/app.sh" ] \ No newline at end of file diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 00000000..58424fb3 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} + +if [ ! -f $BASE/jdk*.rpm ] ; then + DOWNLOAD_LINK="http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz" + wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=xxx; oraclelicense=accept-securebackup-cookie" "$DOWNLOAD_LINK" -O $BASE/jdk-8-linux-x64.rpm +fi + +cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ; + +if [ "$1" == "kafka" ] ; then + cp $BASE/../target/canal-kafka-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-server $BASE/ +else + cp $BASE/../target/canal.deployer-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-server $BASE/ +fi diff --git a/docker/image/admin/app.sh b/docker/image/admin/app.sh new file mode 100755 index 00000000..254f7e48 --- /dev/null +++ b/docker/image/admin/app.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e + +source /etc/profile +export JAVA_HOME=/usr/java/latest +export PATH=$JAVA_HOME/bin:$PATH +touch /tmp/start.log +chown admin: /tmp/start.log +chown -R admin: /home/admin/canal-server +host=`hostname -i` + +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} + + +function checkStart() { + local name=$1 + local cmd=$2 + local timeout=$3 + cost=5 + while [ $timeout -gt 0 ]; do + ST=`eval $cmd` + if [ "$ST" == "0" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + elif [ "$ST" == "" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + else + break + fi + done + echo "start $name successful" +} + + +function start_canal() { + echo "start canal ..." + serverPort=`perl -le 'print $ENV{"canal.port"}'` + if [ -z "$serverPort" ] ; then + serverPort=11111 + fi + + destination=`perl -le 'print $ENV{"canal.destinations"}'` + if [[ "$destination" =~ ',' ]]; then + echo "multi destination:$destination is not support" + exit 1; + else + mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination + fi + su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1' + sleep 5 + #check start + checkStart "canal" "nc 127.0.0.1 $serverPort -w 1 -z | wc -l" 30 +} + +function stop_canal() { + echo "stop canal" + su admin -c 'cd /home/admin/canal-server/bin/ && sh stop.sh 1>>/tmp/start.log 2>&1' + echo "stop canal successful ..." +} + +echo "==> START ..." + +start_canal + +echo "==> START SUCCESSFUL ..." + +tail -f /dev/null & +# wait TERM signal +waitterm + +echo "==> STOP" + +stop_canal + +echo "==> STOP SUCCESSFUL ..." diff --git a/docker/image/admin/bin/clean_log b/docker/image/admin/bin/clean_log new file mode 100644 index 00000000..df183614 --- /dev/null +++ b/docker/image/admin/bin/clean_log @@ -0,0 +1,2 @@ +# cron clean log once per minute +*/2 * * * * admin /home/admin/bin/clean_log.sh >>/tmp/clean_log.log 2>&1 diff --git a/docker/image/admin/bin/clean_log.sh b/docker/image/admin/bin/clean_log.sh new file mode 100755 index 00000000..9d0a0bef --- /dev/null +++ b/docker/image/admin/bin/clean_log.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Global Settings +PATH="$HOME/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/X11R6/bin:/root/bin" +export PATH + +CUTOFF="85" +#获取磁盘使用率最高的分区 +USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) +before=$USAGE + +baseClean(){ + #删除tmp目录15天前的文件。 + #更新文档时间戳 + if [ -d /tmp/hsperfdata_admin ] + then + touch /tmp/hsperfdata_admin + touch /tmp/hsperfdata_admin/* + fi + + find /tmp/ -type f -mtime +15 | xargs -t rm -rf >/dev/null 2>&1 + + + now=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + echo "before:$before; now:$now" +} + +CANAL_DIR="/home/admin/canal-server/logs" +if [[ -d $CANAL_DIR ]]; then + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 90 ]]; then + find $CANAL_DIR -type f -mtime +7 | xargs rm -rf {} + fi + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 80 ]]; then + find $CANAL_DIR -type f -mtime +3 | xargs rm -rf {} + fi + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 80 ]]; then + find $CANAL_DIR -type d -empty -mtime +3 | grep -v canal | xargs rm -rf {} + find $CANAL_DIR -type f -iname '*.tmp' | xargs rm -rf {} + fi + baseClean + exit 0 +fi \ No newline at end of file diff --git a/docker/image/admin/health.sh b/docker/image/admin/health.sh new file mode 100755 index 00000000..17a89740 --- /dev/null +++ b/docker/image/admin/health.sh @@ -0,0 +1,13 @@ +#!/bin/sh +CHECK_URL="http://127.0.0.1:8080/metrics" +CHECK_POINT="success" +CHECK_COUNT=`curl -s --connect-timeout 7 --max-time 7 $CHECK_URL | grep -c $CHECK_POINT` +if [ $CHECK_COUNT -eq 0 ]; then + echo "[FAILED]" + status=0 + error=1 +else + echo "[ OK ]" + status=1 + error=0 +fi \ No newline at end of file diff --git a/docker/image/alidata/bin/exec_rc_local.sh b/docker/image/alidata/bin/exec_rc_local.sh new file mode 100755 index 00000000..7cc8aa3f --- /dev/null +++ b/docker/image/alidata/bin/exec_rc_local.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ "${SKIP_EXEC_RC_LOCAL}" = "YES" ] ; then + echo "skip /etc/rc.local: SKIP_EXEC_RC_LOCAL=${SKIP_EXEC_RC_LOCAL}" + exit +fi + +if [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] ; then + echo "skip /etc/rc.local: DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" + exit +fi \ No newline at end of file diff --git a/docker/image/alidata/bin/lark-wait b/docker/image/alidata/bin/lark-wait new file mode 100755 index 00000000..db7be708 --- /dev/null +++ b/docker/image/alidata/bin/lark-wait @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +chown admin: -R /home/admin/ +source /alidata/lib/proc.sh +waitterm diff --git a/docker/image/alidata/bin/main.sh b/docker/image/alidata/bin/main.sh new file mode 100755 index 00000000..a57551a7 --- /dev/null +++ b/docker/image/alidata/bin/main.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +[ -n "${DOCKER_DEPLOY_TYPE}" ] || DOCKER_DEPLOY_TYPE="VM" +echo "DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" + +# run init scripts +for e in $(ls /alidata/init/*) ; do + [ -x "${e}" ] || continue + echo "==> INIT $e" + $e + echo "==> EXIT CODE: $?" +done + +echo "==> INIT DEFAULT" +service sshd start +service crond start + +#echo "check hostname -i: `hostname -i`" +#hti_num=`hostname -i|awk '{print NF}'` +#if [ $hti_num -gt 1 ];then +# echo "hostname -i result error:`hostname -i`" +# exit 120 +#fi + +echo "==> INIT DONE" +echo "==> RUN ${*}" +exec "${@}" \ No newline at end of file diff --git a/docker/image/alidata/init/02init-sshd.sh b/docker/image/alidata/init/02init-sshd.sh new file mode 100755 index 00000000..118ee553 --- /dev/null +++ b/docker/image/alidata/init/02init-sshd.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# set port +if [ -z "${SSHD_PORT}" ] ; then + SSHD_PORT=22 + [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] && SSHD_PORT=2222 +fi + +sed -r -i '/^OPTIONS=/ d' /etc/sysconfig/sshd +echo 'OPTIONS="-p '"${SSHD_PORT}"'"' >> /etc/sysconfig/sshd + +# set admin ssh pulic key +if [ "${USE_ADMIN_PASSAGE}" = "YES" ] ; then + echo "set admin passage" + mkdir -p /home/admin/.ssh + chown admin:admin /home/admin/.ssh + chown admin:admin /home/admin/.ssh/authorized_keys + chmod 644 /home/admin/.ssh/authorized_keys +fi diff --git a/docker/image/alidata/init/fix-hosts.py b/docker/image/alidata/init/fix-hosts.py new file mode 100755 index 00000000..4ee8abb5 --- /dev/null +++ b/docker/image/alidata/init/fix-hosts.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +#****************************************************************# +# Create Date: 2017-01-06 17:58 +#***************************************************************# + +import socket +import shutil +from time import gmtime, strftime + +# get host_name +host_name = socket.gethostname() +tmp_file = "/tmp/.lark-fix-host.hosts" +host_file = "/etc/hosts" +bak_file_name = "/tmp/hosts-fix-bak.%s" % ( strftime("%Y-%m-%d_%H-%M-%S", gmtime()) ) + +# load /etc/hosts file context +FH = open(host_file,"r") +file_lines = [ i.rstrip() for i in FH.readlines()] +FH.close() +file_lines_reverse = file_lines[::-1] +new_lines = [] +bad_lines = [] +last_match_line = "" + +for line in file_lines_reverse: + if line.find(host_name) < 0: # 不匹配的行直接跳过 + new_lines.append(line + "\n") + continue + + cols = line.split() + new_cols = [] + if cols[0].startswith("#"): # 跳过已经注释掉的行 + new_lines.append(line + "\n") + continue + for col in cols: + if not col == host_name: # 跳过不匹配的列 + new_cols.append(col) + continue + + if cols[0] == "127.0.0.1": # 如果第一列是 127.0.0.1 就跳过匹配的列, 防止 hostname -i 返回 127.0.0.1 + continue + + # 如果已经发现过匹配的列, 就丢掉重复的列 + if not len(last_match_line) == 0: + continue + + new_cols.append(col) + last_match_line = line + + # 跳过 xx.xx.xx.xx hostname 这样的重复列 + if len(new_cols) == 1: + continue + + new_l = "%s\n" % " ".join(new_cols) + new_lines.append(new_l) + +# save tmp hosts + +FH2=file(tmp_file,"w+") +FH2.writelines( new_lines[::-1]) +FH2.close() + +# mv to /etc/hosts +shutil.copy(host_file, bak_file_name) +shutil.move(tmp_file, host_file) diff --git a/docker/image/alidata/lib/proc.sh b/docker/image/alidata/lib/proc.sh new file mode 100644 index 00000000..40ddff93 --- /dev/null +++ b/docker/image/alidata/lib/proc.sh @@ -0,0 +1,40 @@ +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} diff --git a/docker/run.sh b/docker/run.sh new file mode 100644 index 00000000..7c56e2ba --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +function usage() { + echo "Usage:" + echo " run.sh [CONFIG]" + echo "example:" + echo " run.sh -e canal.instance.master.address=127.0.0.1:3306 \\" + echo " -e canal.instance.dbUsername=canal \\" + echo " -e canal.instance.dbPassword=canal \\" + echo " -e canal.instance.connectionCharset=UTF-8 \\" + echo " -e canal.instance.tsdb.enable=true \\" + echo " -e canal.instance.gtidon=false \\" + echo " -e canal.instance.filter.regex=.*\\..* " + exit +} + +function check_port() { + local port=$1 + local TL=$(which telnet) + if [ -f $TL ]; then + data=`echo quit | telnet 127.0.0.1 $port| grep -ic connected` + echo $data + return + fi + + local NC=$(which nc) + if [ -f $NC ]; then + data=`nc -z -w 1 127.0.0.1 $port | grep -ic succeeded` + echo $data + return + fi + echo "0" + return +} + +function getMyIp() { + case "`uname`" in + Darwin) + myip=`echo "show State:/Network/Global/IPv4" | scutil | grep PrimaryInterface | awk '{print $3}' | xargs ifconfig | grep inet | grep -v inet6 | awk '{print $2}'` + ;; + *) + myip=`ip route get 1 | awk '{print $NF;exit}'` + ;; + esac + echo $myip +} + +NET_MODE="" +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + NET_MODE="--net=host" + ;; + *) + NET_MODE="--net=host" + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} +if [ $# -eq 0 ]; then + usage +elif [ "$1" == "-h" ] ; then + usage +elif [ "$1" == "help" ] ; then + usage +fi + +DATA="$BASE/data" +mkdir -p $DATA +CONFIG=${@:1} +#VOLUMNS="-v $DATA:/home/admin/canal-server/logs" +PORTLIST="8000 8080 2222 11111" +PORTS="" +for PORT in $PORTLIST ; do + #exist=`check_port $PORT` + exist="0" + if [ "$exist" == "0" ]; then + PORTS="$PORTS -p $PORT:$PORT" + else + echo "port $PORT is used , pls check" + exit 1 + fi +done + +MEMORY="-m 4096m" +LOCALHOST=`getMyIp` +cmd="docker run -it -h $LOCALHOST $CONFIG --name=canal-server $VOLUMNS $NET_MODE $PORTS $MEMORY canal/canal-server" +echo $cmd +eval $cmd \ No newline at end of file diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 29b96852..0f01cbd5 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -32,7 +32,7 @@ public class MysqlConnector { private String password; private byte charsetNumber = 33; - private String defaultSchema = "retl"; + private String defaultSchema = "test"; private int soTimeout = 30 * 1000; private int connTimeout = 5 * 1000; private int receiveBufferSize = 16 * 1024; From 872dad8ce82e21e3acfdb5d3f596e81e52b44f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Sat, 4 Aug 2018 10:37:55 +0800 Subject: [PATCH 09/21] fixed issue #801 , optimizer docker --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e10085b..e44796cb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,9 +45,7 @@ RUN \ echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ - /bin/mv /home/admin/bin/canal-check /etc/cron.d && \ /bin/mv /home/admin/bin/clean_log /etc/cron.d && \ - chmod +x /home/admin/bin/canal-process-check.py && \ mkdir -p /home/admin/canal-server && \ tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ From 3c716e2a436f729f1394e9a92e86be2e9095da1e Mon Sep 17 00:00:00 2001 From: qmzhang <578967933@qq.com> Date: Sun, 5 Aug 2018 19:40:14 +0800 Subject: [PATCH 10/21] =?UTF-8?q?instance=E7=94=B1manager=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E6=97=B6=EF=BC=8CCanalParameter#MetaMode=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E6=9C=AC=E5=9C=B0meta.dat=E5=AD=98=E5=82=A8=E4=BD=8D?= =?UTF-8?q?=E7=82=B9=E4=BF=A1=E6=81=AF=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/CanalInstanceWithManager.java | 21 +++----- .../manager/model/CanalParameter.java | 26 ++++++++- .../CanalServerWithEmbedded_FileModeTest.java | 53 +++++++++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index bca5ba13..35e3235a 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import com.alibaba.otter.canal.meta.FileMixedMetaManager; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,13 +21,7 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.instance.core.AbstractCanalInstance; import com.alibaba.otter.canal.instance.manager.model.Canal; import com.alibaba.otter.canal.instance.manager.model.CanalParameter; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; import com.alibaba.otter.canal.meta.MemoryMetaManager; import com.alibaba.otter.canal.meta.PeriodMixedMetaManager; import com.alibaba.otter.canal.meta.ZooKeeperMetaManager; @@ -37,12 +32,7 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; -import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager; -import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager; -import com.alibaba.otter.canal.parse.index.MetaLogPositionManager; -import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager; -import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager; +import com.alibaba.otter.canal.parse.index.*; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.sink.entry.EntryEventSink; @@ -120,6 +110,11 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(getZkclientx()); ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager); + } else if (mode.isLocalFile()){ + FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager(); + fileMixedMetaManager.setDataDir(parameters.getDataDir()); + fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod()); + metaManager = fileMixedMetaManager; } else { throw new CanalException("unsupport MetaMode for " + mode); } diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index 16c929ca..befc8e2f 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -28,8 +28,10 @@ public class CanalParameter implements Serializable { private Long zkClusterId; // zk集群id，为管理方便 private List zkClusters; // zk集群地址 + private String dataDir = "../conf"; // 默认本地文件数据的目录默认是conf // meta相关参数 private MetaMode metaMode = MetaMode.MEMORY; // meta机制 + private Integer metaFileFlushPeriod = 1000; // meta刷新间隔 // storage存储 private Integer transactionSize = 1024; // 支持处理的transaction事务大小 @@ -243,7 +245,9 @@ public class CanalParameter implements Serializable { /** 文件存储模式 */ ZOOKEEPER, /** 混合模式，内存+文件 */ - MIXED; + MIXED, + /** 本地文件存储模式*/ + LOCAL_FILE; public boolean isMemory() { return this.equals(MetaMode.MEMORY); @@ -256,6 +260,10 @@ public class CanalParameter implements Serializable { public boolean isMixed() { return this.equals(MetaMode.MIXED); } + + public boolean isLocalFile(){ + return this.equals(MetaMode.LOCAL_FILE); + } } public static enum IndexMode { @@ -390,6 +398,22 @@ public class CanalParameter implements Serializable { return storageMode; } + public String getDataDir() { + return dataDir; + } + + public void setDataDir(String dataDir) { + this.dataDir = dataDir; + } + + public Integer getMetaFileFlushPeriod() { + return metaFileFlushPeriod; + } + + public void setMetaFileFlushPeriod(Integer metaFileFlushPeriod) { + this.metaFileFlushPeriod = metaFileFlushPeriod; + } + public void setStorageMode(StorageMode storageMode) { this.storageMode = storageMode; } diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java new file mode 100644 index 00000000..e986b6ff --- /dev/null +++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java @@ -0,0 +1,53 @@ +package com.alibaba.otter.canal.server; + +import java.net.InetSocketAddress; +import java.util.Arrays; + +import com.alibaba.otter.canal.instance.manager.model.Canal; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; + +public class CanalServerWithEmbedded_FileModeTest extends BaseCanalServerWithEmbededTest { + + protected Canal buildCanal() { + Canal canal = new Canal(); + canal.setId(1L); + canal.setName(DESTINATION); + canal.setDesc("my standalone server test "); + + CanalParameter parameter = new CanalParameter(); + + parameter.setMetaMode(MetaMode.LOCAL_FILE); + parameter.setDataDir("./conf"); + parameter.setMetaFileFlushPeriod(1000); + parameter.setHaMode(HAMode.HEARTBEAT); + parameter.setIndexMode(IndexMode.MEMORY_META_FAILBACK); + + parameter.setStorageMode(StorageMode.MEMORY); + parameter.setMemoryStorageBufferSize(32 * 1024); + + parameter.setSourcingType(SourcingType.MYSQL); + parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306), + new InetSocketAddress(MYSQL_ADDRESS, 3306))); + parameter.setDbUsername(USERNAME); + parameter.setDbPassword(PASSWORD); + parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}", + "{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}")); + + parameter.setSlaveId(1234L); + + parameter.setDefaultConnectionTimeoutInSeconds(30); + parameter.setConnectionCharset("UTF-8"); + parameter.setConnectionCharsetNumber((byte) 33); + parameter.setReceiveBufferSize(8 * 1024); + parameter.setSendBufferSize(8 * 1024); + + parameter.setDetectingEnable(false); + parameter.setDetectingIntervalInSeconds(10); + parameter.setDetectingRetryTimes(3); + parameter.setDetectingSQL(DETECTING_SQL); + + canal.setCanalParameter(parameter); + return canal; + } +} \ No newline at end of file From 8b186d1330fa7418cc167b74a8610f75353480d2 Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Mon, 6 Aug 2018 08:12:59 +0800 Subject: [PATCH 11/21] canal server metrics refactor. --- .../common/utils/SerializedLongAdder.java | 31 ++++ .../parse/inbound/AbstractEventParser.java | 3 + .../mysql/AbstractMysqlEventParser.java | 44 +++--- .../parse/inbound/mysql/MysqlConnection.java | 38 +++-- .../parse/inbound/mysql/MysqlEventParser.java | 15 +- .../mysql/MysqlMultiStageCoprocessor.java | 30 +++- .../prometheus/CanalInstanceExports.java | 49 +++++-- .../canal/prometheus/CanalServerExports.java | 4 - .../canal/prometheus/PrometheusService.java | 7 + .../prometheus/impl/EntrySinkCollector.java | 45 ++++++ .../impl/InboundThroughputAspect.java | 79 ----------- .../impl/InstanceMetaCollector.java | 2 +- .../prometheus/impl/MemoryStoreCollector.java | 9 +- .../prometheus/impl/MysqlParserCollector.java | 74 ++++++++++ .../impl/OutboundThroughputAspect.java | 80 ----------- ...PrometheusCanalEventDownStreamHandler.java | 2 +- .../PrometheusClientInstanceProfiler.java | 134 ++++++++++++++++++ ...ometheusClientInstanceProfilerFactory.java | 15 ++ .../src/main/resources/META-INF/aop.xml | 11 -- .../netty/CanalServerWithNettyProfiler.java | 58 +++++--- .../server/netty/ClientInstanceProfiler.java | 3 +- .../netty/ClientInstanceProfilerFactory.java | 2 +- .../server/netty/handler/SessionHandler.java | 1 + .../listener/ChannelFutureAggregator.java | 62 +++++--- .../canal/sink/entry/EntryEventSink.java | 13 +- 25 files changed, 539 insertions(+), 272 deletions(-) create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java delete mode 100644 prometheus/src/main/resources/META-INF/aop.xml diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java new file mode 100644 index 00000000..4d86a99e --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.common.utils; + +/** + * 用于dump记录counter + * 这种场景下， + * 写操作>>读操作 + * dump线程为串行，读取counter时，并不严格要求最新值可见，使用普通long即可。 + * + * @author Chuanyi Li + */ +public class SerializedLongAdder { + + private long value = 0; + + public SerializedLongAdder(long initial) { + this.value = initial; + } + + public void add(long x) { + value += x; + } + + public void increment() { + value ++; + } + + public long get() { + return value; + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index 7f7d39ac..ff92dd9d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -8,6 +8,7 @@ import java.util.TimerTask; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang.math.RandomUtils; @@ -95,6 +96,7 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle protected int parallelBufferSize = 256; // 必须为2的幂 protected MultiStageCoprocessor multiStageCoprocessor; + protected abstract BinlogParser buildParser(); protected abstract ErosaConnection buildErosaConnection(); @@ -615,4 +617,5 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle this.parallelBufferSize = parallelBufferSize; } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 8188cf83..5e3c6b02 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.nio.charset.Charset; +import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,21 +20,22 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition; public abstract class AbstractMysqlEventParser extends AbstractEventParser { - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - protected static final long BINLOG_START_OFFEST = 4L; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + protected static final long BINLOG_START_OFFEST = 4L; - protected boolean enableTsdb = false; - protected String tsdbSpringXml; - protected TableMetaTSDB tableMetaTSDB; + protected boolean enableTsdb = false; + protected String tsdbSpringXml; + protected TableMetaTSDB tableMetaTSDB; // 编码信息 - protected byte connectionCharsetNumber = (byte) 33; - protected Charset connectionCharset = Charset.forName("UTF-8"); - protected boolean filterQueryDcl = false; - protected boolean filterQueryDml = false; - protected boolean filterQueryDdl = false; - protected boolean filterRows = false; - protected boolean filterTableError = false; - protected boolean useDruidDdlFilter = true; + protected byte connectionCharsetNumber = (byte) 33; + protected Charset connectionCharset = Charset.forName("UTF-8"); + protected boolean filterQueryDcl = false; + protected boolean filterQueryDml = false; + protected boolean filterQueryDdl = false; + protected boolean filterRows = false; + protected boolean filterTableError = false; + protected boolean useDruidDdlFilter = true; + private final SerializedLongAdder eventsPublishBlockingTime = new SerializedLongAdder(0L); protected BinlogParser buildParser() { LogEventConvert convert = new LogEventConvert(); @@ -119,11 +121,13 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { } protected MultiStageCoprocessor buildMultiStageCoprocessor() { - return new MysqlMultiStageCoprocessor(parallelBufferSize, - parallelThreadSize, - (LogEventConvert) binlogParser, - transactionBuffer, - destination); + MysqlMultiStageCoprocessor mysqlMultiStageCoprocessor = new MysqlMultiStageCoprocessor(parallelBufferSize, + parallelThreadSize, + (LogEventConvert) binlogParser, + transactionBuffer, + destination); + mysqlMultiStageCoprocessor.setEventsPublishBlockingTime(eventsPublishBlockingTime); + return mysqlMultiStageCoprocessor; } // ============================ setter / getter ========================= @@ -188,4 +192,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { } } + public SerializedLongAdder getEventsPublishBlockingTime() { + return this.eventsPublishBlockingTime; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index d01a210e..c125dbd3 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -8,6 +8,7 @@ import java.nio.charset.Charset; import java.util.List; import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,18 +38,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent; public class MysqlConnection implements ErosaConnection { - private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class); + private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class); - private MysqlConnector connector; - private long slaveId; - private Charset charset = Charset.forName("UTF-8"); - private BinlogFormat binlogFormat; - private BinlogImage binlogImage; + private MysqlConnector connector; + private long slaveId; + private Charset charset = Charset.forName("UTF-8"); + private BinlogFormat binlogFormat; + private BinlogImage binlogImage; // tsdb releated - private AuthenticationInfo authInfo; - protected int connTimeout = 5 * 1000; // 5秒 - protected int soTimeout = 60 * 60 * 1000; // 1小时 + private AuthenticationInfo authInfo; + protected int connTimeout = 5 * 1000; // 5秒 + protected int soTimeout = 60 * 60 * 1000; // 1小时 + private SerializedLongAdder receivedBinlogBytes; public MysqlConnection(){ } @@ -124,6 +126,7 @@ public class MysqlConnection implements ErosaConnection { decoder.handle(LogEvent.XID_EVENT); LogContext context = new LogContext(); while (fetcher.fetch()) { + accumulateReceivedBytes(fetcher.limit()); LogEvent event = null; event = decoder.decode(fetcher, context); @@ -146,6 +149,7 @@ public class MysqlConnection implements ErosaConnection { LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); LogContext context = new LogContext(); while (fetcher.fetch()) { + accumulateReceivedBytes(fetcher.limit()); LogEvent event = null; event = decoder.decode(fetcher, context); @@ -174,6 +178,7 @@ public class MysqlConnection implements ErosaConnection { LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); LogContext context = new LogContext(); while (fetcher.fetch()) { + accumulateReceivedBytes(fetcher.limit()); LogEvent event = null; event = decoder.decode(fetcher, context); @@ -204,6 +209,7 @@ public class MysqlConnection implements ErosaConnection { try { fetcher.start(connector.getChannel()); while (fetcher.fetch()) { + accumulateReceivedBytes(fetcher.limit()); LogBuffer buffer = fetcher.duplicate(); fetcher.consume(fetcher.limit()); if (!coprocessor.publish(buffer)) { @@ -230,6 +236,7 @@ public class MysqlConnection implements ErosaConnection { try { fetcher.start(connector.getChannel()); while (fetcher.fetch()) { + accumulateReceivedBytes(fetcher.limit()); LogBuffer buffer = fetcher.duplicate(); fetcher.consume(fetcher.limit()); if (!coprocessor.publish(buffer)) { @@ -334,7 +341,6 @@ public class MysqlConnection implements ErosaConnection { *
  • net_read_timeout
    • * * - * @param channel * @throws IOException */ private void updateSettings() throws IOException { @@ -453,6 +459,14 @@ public class MysqlConnection implements ErosaConnection { } } + private void accumulateReceivedBytes(long x) { + if (receivedBinlogBytes != null) { + receivedBinlogBytes.add(x); + } + } + + + public static enum BinlogFormat { STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED"); @@ -592,4 +606,8 @@ public class MysqlConnection implements ErosaConnection { this.authInfo = authInfo; } + public void setReceivedBinlogBytes(SerializedLongAdder receivedBinlogBytes) { + this.receivedBinlogBytes = receivedBinlogBytes; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 6c2b9d4d..5baad994 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.TimerTask; import java.util.concurrent.atomic.AtomicLong; +import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import org.springframework.util.CollectionUtils; @@ -69,8 +70,11 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE private BinlogImage[] supportBinlogImages; // 支持的binlogImage,如果设置会执行强校验 // update by yishun.chen,特殊异常处理参数 - private int dumpErrorCount = 0; // binlogDump失败异常计数 - private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 + private int dumpErrorCount = 0; // binlogDump失败异常计数 + private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 + + // instance received binlog bytes + private final SerializedLongAdder receivedBinlogBytes = new SerializedLongAdder(0L); protected ErosaConnection buildErosaConnection() { return buildMysqlConnection(this.runningInfo); @@ -314,6 +318,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE connection.getConnector().setSendBufferSize(sendBufferSize); connection.getConnector().setSoTimeout(defaultConnectionTimeoutInSeconds * 1000); connection.setCharset(connectionCharset); + connection.setReceivedBinlogBytes(receivedBinlogBytes); // 随机生成slaveId if (this.slaveId <= 0) { this.slaveId = generateUniqueServerId(); @@ -908,4 +913,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE this.dumpErrorCountThreshold = dumpErrorCountThreshold; } + + + public SerializedLongAdder getReceivedBinlogBytes() { + return this.receivedBinlogBytes; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 281a05f2..e3206e3e 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -4,6 +4,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.LockSupport; +import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; @@ -52,6 +53,7 @@ import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; */ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implements MultiStageCoprocessor { + private static final int maxFullTimes = 10; private LogEventConvert logEventConvert; private EventTransactionBuffer transactionBuffer; private ErosaConnection connection; @@ -63,6 +65,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement private ExecutorService stageExecutor; private String destination; private volatile CanalParseException exception; + private SerializedLongAdder eventsPublishBlockingTime; public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert, EventTransactionBuffer transactionBuffer, String destination){ @@ -161,6 +164,8 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement throw exception; } boolean interupted = false; + long blockingStart = 0L; + int fullTimes = 0; do { try { long next = disruptorMsgBuffer.tryNext(); @@ -170,16 +175,34 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement event.setBinlogFileName(binlogFileName); } disruptorMsgBuffer.publish(next); + if (fullTimes > 0) { + eventsPublishBlockingTime.add(System.nanoTime() - blockingStart); + } break; } catch (InsufficientCapacityException e) { + if (fullTimes == 0) { + blockingStart = System.nanoTime(); + } // park - LockSupport.parkNanos(1L); + //LockSupport.parkNanos(1L); + applyWait(++fullTimes); interupted = Thread.interrupted(); } } while (!interupted && isStart()); return isStart(); } + // 处理无数据的情况，避免空循环挂死 + private void applyWait(int fullTimes) { + int newFullTimes = fullTimes > maxFullTimes ? maxFullTimes : fullTimes; + if (fullTimes <= 3) { // 3次以内 + Thread.yield(); + } else { // 超过3次，最多只sleep 1ms + LockSupport.parkNanos(100 * 1000L * newFullTimes); + } + + } + @Override public void reset() { if (isStart()) { @@ -189,6 +212,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement start(); } + private class SimpleParserStage implements EventHandler, LifecycleAware { private LogDecoder decoder; @@ -427,4 +451,8 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement this.connection = connection; } + public void setEventsPublishBlockingTime(SerializedLongAdder eventsPublishBlockingTime) { + this.eventsPublishBlockingTime = eventsPublishBlockingTime; + } + } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java index 8acf40b8..19759154 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -1,9 +1,7 @@ package com.alibaba.otter.canal.prometheus; import com.alibaba.otter.canal.instance.core.CanalInstance; -import com.alibaba.otter.canal.prometheus.impl.InstanceMetaCollector; -import com.alibaba.otter.canal.prometheus.impl.MemoryStoreCollector; -import com.alibaba.otter.canal.prometheus.impl.PrometheusCanalEventDownStreamHandler; +import com.alibaba.otter.canal.prometheus.impl.*; import com.alibaba.otter.canal.sink.CanalEventSink; import com.alibaba.otter.canal.sink.entry.EntryEventSink; import com.alibaba.otter.canal.store.CanalStoreException; @@ -15,30 +13,31 @@ import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; +import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.profiler; + /** * @author Chuanyi Li */ public class CanalInstanceExports { - private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); - - public static final String[] labels = {"destination"}; - - public static final List labelList = Collections.singletonList(labels[0]); - + private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); + public static final String DESTINATION = "destination"; + public static final String[] DEST_LABELS = {DESTINATION}; + public static final List DEST_LABEL_LIST = Collections.singletonList(DESTINATION); private final String destination; - private Collector storeCollector; - private Collector delayCollector; - private Collector metaCollector; + private Collector sinkCollector; + private Collector parserCollector; private CanalInstanceExports(CanalInstance instance) { this.destination = instance.getDestination(); - initDelayGauge(instance); + initEventsMetrics(instance); initStoreCollector(instance); initMetaCollector(instance); + initSinkCollector(instance); + initParserCollector(instance); } @@ -48,6 +47,7 @@ public class CanalInstanceExports { } void register() { + profiler().start(destination); if (delayCollector != null) { delayCollector.register(); } @@ -57,9 +57,16 @@ public class CanalInstanceExports { if (metaCollector != null) { metaCollector.register(); } + if (sinkCollector != null) { + sinkCollector.register(); + } + if (parserCollector != null) { + parserCollector.register(); + } } void unregister() { + profiler().stop(destination); if (delayCollector != null) { CollectorRegistry.defaultRegistry.unregister(delayCollector); } @@ -69,9 +76,15 @@ public class CanalInstanceExports { if (metaCollector != null) { CollectorRegistry.defaultRegistry.unregister(metaCollector); } + if (sinkCollector != null) { + CollectorRegistry.defaultRegistry.unregister(sinkCollector); + } + if (parserCollector != null) { + CollectorRegistry.defaultRegistry.unregister(parserCollector); + } } - private void initDelayGauge(CanalInstance instance) { + private void initEventsMetrics(CanalInstance instance) { CanalEventSink sink = instance.getEventSink(); if (sink instanceof EntryEventSink) { EntryEventSink entryEventSink = (EntryEventSink) sink; @@ -95,4 +108,12 @@ public class CanalInstanceExports { private void initMetaCollector(CanalInstance instance) { metaCollector = new InstanceMetaCollector(instance); } + + private void initSinkCollector(CanalInstance instance) { + sinkCollector = new EntrySinkCollector(instance.getEventSink(), instance.getDestination()); + } + + private void initParserCollector(CanalInstance instance) { + parserCollector = new MysqlParserCollector(instance.getEventParser(), instance.getDestination()); + } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java index 8eef79ba..ded44fb8 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java @@ -1,7 +1,5 @@ package com.alibaba.otter.canal.prometheus; -import com.alibaba.otter.canal.prometheus.impl.InboundThroughputAspect; -import com.alibaba.otter.canal.prometheus.impl.OutboundThroughputAspect; /** * @author Chuanyi Li @@ -12,8 +10,6 @@ public class CanalServerExports { public static synchronized void initialize() { if (!initialized) { - InboundThroughputAspect.getCollector().register(); - OutboundThroughputAspect.getCollector().register(); initialized = true; } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java index c7942235..2aa7dcd0 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -1,6 +1,8 @@ package com.alibaba.otter.canal.prometheus; import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.prometheus.impl.PrometheusClientInstanceProfilerFactory; +import com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler; import com.alibaba.otter.canal.spi.CanalMetricsService; import io.prometheus.client.exporter.HTTPServer; import io.prometheus.client.hotspot.DefaultExports; @@ -11,6 +13,9 @@ import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.DISABLED; +import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.profiler; + /** * @author Chuanyi Li */ @@ -50,6 +55,7 @@ public class PrometheusService implements CanalMetricsService { DefaultExports.initialize(); // Canal server level exports CanalServerExports.initialize(); + profiler().setInstanceProfilerFactory(new PrometheusClientInstanceProfilerFactory()); } catch (Throwable t) { logger.warn("Unable to initialize server exports.", t); } @@ -66,6 +72,7 @@ public class PrometheusService implements CanalMetricsService { for (CanalInstanceExports ie : exports.values()) { ie.unregister(); } + profiler().setInstanceProfilerFactory(DISABLED); if (server != null) { server.stop(); } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java new file mode 100644 index 00000000..54775842 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java @@ -0,0 +1,45 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.sink.CanalEventSink; +import com.alibaba.otter.canal.sink.entry.EntryEventSink; +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; + +/** + * @author Chuanyi Li + */ +public class EntrySinkCollector extends Collector { + + private static final String SINK_BLOCKING_TIME = "canal_instance_sink_blocking_time"; + private final String sinkBlockTimeHelp; + private final List destLabelValues; + private final AtomicLong eventsSinkBlockingTime; + + + public EntrySinkCollector(CanalEventSink sink, String destination) { + if (!(sink instanceof EntryEventSink)) { + throw new IllegalArgumentException("CanalEventSink must be EntryEventSink"); + } + this.destLabelValues = Collections.singletonList(destination); + EntryEventSink entrySink = (EntryEventSink) sink; + this.sinkBlockTimeHelp = "Sink blocking time of instance " + destination; + eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime(); + } + + @Override public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME, + sinkBlockTimeHelp, DEST_LABEL_LIST); + blockingCounter.addMetric(destLabelValues, eventsSinkBlockingTime.doubleValue()); + mfs.add(blockingCounter); + return mfs; + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java deleted file mode 100644 index eabc256c..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.alibaba.otter.canal.prometheus.impl; - -import io.prometheus.client.Collector; -import io.prometheus.client.CounterMetricFamily; -import org.aspectj.lang.annotation.After; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.jctools.maps.ConcurrentAutoTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author Chuanyi Li - */ -@Aspect -public class InboundThroughputAspect { - - private static final Logger logger = LoggerFactory.getLogger(InboundThroughputAspect.class); - - /** - * Support highly scalable counters - * @see ConcurrentAutoTable - */ - private static final ConcurrentAutoTable total = new ConcurrentAutoTable(); - - private static final Collector collector = new InboundThroughputCollector(); - - public static Collector getCollector() { - return collector; - } - - @Pointcut("call(byte[] com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))") - public void read() {} - - @Pointcut("call(void com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..)) ") - public void readBytes() {} - - //nested read, just eliminate them. - @Pointcut("withincode(* com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))") - public void nestedCall() {} - - @After("read() && !nestedCall() && args(len, ..)") - public void recordRead(int len) { - accumulateBytes(len); - } - - @After("readBytes() && !nestedCall() && args(.., len, timeout)") - public void recordReadBytes(int len, int timeout) { - accumulateBytes(len); - } - - private void accumulateBytes(int count) { - try { - total.add(count); - } catch (Throwable t) { - //Catch every Throwable, rather than break the business logic. - logger.warn("Error while accumulate inbound bytes.", t); - } - } - - public static class InboundThroughputCollector extends Collector { - - private InboundThroughputCollector() {} - - @Override - public List collect() { - List mfs = new ArrayList(); - CounterMetricFamily bytes = new CounterMetricFamily("canal_net_inbound_bytes", - "Total socket inbound bytes of canal server.", - total.get()); - mfs.add(bytes); - return mfs; - } - } - -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java index 5562b8fd..2e8b7fc4 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java @@ -55,7 +55,7 @@ public class InstanceMetaCollector extends Collector { List subs = metaManager.listAllSubscribeInfo(destination); GaugeMetricFamily subscriptions = new GaugeMetricFamily( "canal_instance_subscription", - subsHelp, CanalInstanceExports.labelList); + subsHelp, CanalInstanceExports.DEST_LABEL_LIST); subscriptions.addMetric(Arrays.asList(destination), subs.size()); mfs.add(subscriptions); } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java index 79fdca4a..4c212c4c 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java @@ -20,15 +20,10 @@ import java.util.concurrent.atomic.AtomicLong; public class MemoryStoreCollector extends Collector { private static final Class clazz = MemoryEventStoreWithBuffer.class; - private final String destination; - private final AtomicLong putSequence; - private final AtomicLong ackSequence; - private final String putHelp; - private final String ackHelp; public MemoryStoreCollector(CanalEventStore store, String destination) { @@ -47,11 +42,11 @@ public class MemoryStoreCollector extends Collector { public List collect() { List mfs = new ArrayList(); CounterMetricFamily put = new CounterMetricFamily("canal_instance_store_produce_seq", - putHelp, Arrays.asList(CanalInstanceExports.labels)); + putHelp, Arrays.asList(CanalInstanceExports.DEST_LABELS)); put.addMetric(Collections.singletonList(destination), putSequence.doubleValue()); mfs.add(put); CounterMetricFamily ack = new CounterMetricFamily("canal_instance_store_consume_seq", - ackHelp, Arrays.asList(CanalInstanceExports.labels)); + ackHelp, Arrays.asList(CanalInstanceExports.DEST_LABELS)); ack.addMetric(Collections.singletonList(destination), ackSequence.doubleValue()); mfs.add(ack); return mfs; diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java new file mode 100644 index 00000000..02d35209 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java @@ -0,0 +1,74 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.common.utils.SerializedLongAdder; +import com.alibaba.otter.canal.parse.CanalEventParser; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DESTINATION; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; + +/** + * @author Chuanyi Li + */ +public class MysqlParserCollector extends Collector { + + private static final String PUBLISH_BLOCKING_TIME = "canal_instance_publish_blocking_time"; + private static final String RECEIVED_BINLOG_BYTES = "canal_instance_received_binlog_bytes"; + private static final String PARSER_MODE = "canal_instance_parser_mode"; + private static final String MODE_LABEL = "parallel"; + private final List destLabelValues; + private final List modeLabelValues; + private final SerializedLongAdder receivedBinlogBytes; + private final SerializedLongAdder eventsPublishBlockingTime; + private final String publishBlockingTimeHelp; + private final String receivedBinlogBytesHelp; + private final String modeHelp; + private final List modeLabels; + private final Boolean isParallel; + + public MysqlParserCollector(CanalEventParser parser, String destination) { + if (!(parser instanceof MysqlEventParser)) { + throw new IllegalArgumentException("CanalEventParser must be MysqlEventParser"); + } + this.destLabelValues = Collections.singletonList(destination); + this.eventsPublishBlockingTime = ((MysqlEventParser)parser).getEventsPublishBlockingTime(); + this.publishBlockingTimeHelp = "Blocking time of publishing of instance " + destination; + this.receivedBinlogBytes = ((MysqlEventParser)parser).getReceivedBinlogBytes(); + this.receivedBinlogBytesHelp = "Received binlog bytes of instance" + destination; + this.isParallel = ((MysqlEventParser)parser).isParallel(); + this.modeHelp = "Parser mode of instance" + destination; + this.modeLabels = Arrays.asList(DESTINATION, MODE_LABEL); + this.modeLabelValues = Arrays.asList(destination, isParallel.toString()); + } + + @Override public List collect() { + List mfs = new ArrayList(); + // canal_instance_received_binlog_bytes + CounterMetricFamily bytesCounter = new CounterMetricFamily(RECEIVED_BINLOG_BYTES, + receivedBinlogBytesHelp, DEST_LABEL_LIST); + bytesCounter.addMetric(destLabelValues, receivedBinlogBytes.get()); + mfs.add(bytesCounter); + // canal_instance_parser_mode + GaugeMetricFamily modeGauge = new GaugeMetricFamily(PARSER_MODE, + modeHelp, + modeLabels); + modeGauge.addMetric(modeLabelValues, 1); + mfs.add(modeGauge); + // canal_instance_publish_blocking_time + if (isParallel) { + CounterMetricFamily blockingCounter = new CounterMetricFamily(PUBLISH_BLOCKING_TIME, + publishBlockingTimeHelp, DEST_LABEL_LIST); + blockingCounter.addMetric(destLabelValues, eventsPublishBlockingTime.get()); + mfs.add(blockingCounter); + } + return mfs; + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java deleted file mode 100644 index 60dd808d..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.alibaba.otter.canal.prometheus.impl; - -import io.prometheus.client.Collector; -import io.prometheus.client.CounterMetricFamily; -import org.aspectj.lang.annotation.After; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.jboss.netty.channel.Channel; -import org.jctools.maps.ConcurrentAutoTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import static com.alibaba.otter.canal.server.netty.NettyUtils.HEADER_LENGTH; - -/** - * @author Chuanyi Li - */ -@Aspect -public class OutboundThroughputAspect { - private static final Logger logger = LoggerFactory.getLogger(OutboundThroughputAspect.class); - - /** - * Support highly scalable counters - * @see ConcurrentAutoTable - */ - private static final ConcurrentAutoTable total = new ConcurrentAutoTable(); - - private static final Collector collector = new OutboundThroughputCollector(); - - public static Collector getCollector() { - return collector; - } - - @Pointcut("call(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))") - public void write() {} - - //nested read, just eliminate them. - @Pointcut("withincode(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))") - public void nestedCall() {} - - @After("write() && !nestedCall() && args(ch, bytes, ..)") - public void recordWriteBytes(Channel ch, byte[] bytes) { - if (bytes != null) { - accumulateBytes(HEADER_LENGTH + bytes.length); - } - } - - @After("write() && !nestedCall() && args(ch, buf, ..)") - public void recordWriteBuffer(Channel ch, ByteBuffer buf) { - if (buf != null) { - total.add(HEADER_LENGTH + buf.limit()); - } - } - private void accumulateBytes(int count) { - try { - total.add(count); - } catch (Throwable t) { - //Catch every Throwable, rather than break the business logic. - logger.warn("Error while accumulate inbound bytes.", t); - } - } - - public static class OutboundThroughputCollector extends Collector { - - private OutboundThroughputCollector() {} - - @Override public List collect() { - List mfs = new ArrayList(); - CounterMetricFamily bytes = new CounterMetricFamily("canal_net_outbound_bytes", - "Total socket outbound bytes of canal server.", - total.get()); - mfs.add(bytes); - return mfs; - } - } -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java index c55a7eac..47d4824d 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java @@ -36,7 +36,7 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow GaugeMetricFamily delay = new GaugeMetricFamily( DELAY_NAME, delayHelpName, - CanalInstanceExports.labelList); + CanalInstanceExports.DEST_LABEL_LIST); double d = 0.0; if (latestExecuteTime > 0) { d = now - latestExecuteTime; diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java new file mode 100644 index 00000000..d3413455 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java @@ -0,0 +1,134 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; +import com.alibaba.otter.canal.server.netty.ClientInstanceProfiler; +import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator.ClientRequestResult; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import io.prometheus.client.Histogram; + +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DESTINATION; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS; + +/** + * @author Chuanyi Li + */ +public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler { + + private static final String PACKET_TYPE = "canal_instance_client_packets"; + private static final String OUTBOUND_BYTES = "canal_instance_client_bytes"; + private static final String EMPTY_BATCHES = "canal_instance_client_empty_batches"; + private static final String ERRORS = "canal_instance_client_request_error"; + private static final String LATENCY = "canal_instance_client_request_latency"; + private volatile boolean running = false; + private final String destination; + private final String[] destLabelValues; + private final Counter outboundCounter; + private final Counter packetsCounter; + private final Counter emptyBatchesCounter; + private final Counter errorsCounter; + private final Histogram responseLatency; + + public PrometheusClientInstanceProfiler(String destination) { + this.destination = destination; + this.destLabelValues = new String[]{destination}; + this.outboundCounter = Counter.build() + .labelNames(DEST_LABELS) + .name(OUTBOUND_BYTES) + .help("Send bytes to client of instance " + destination) + .create(); + this.packetsCounter = Counter.build() + .labelNames(new String[]{DESTINATION, "packetType"}) + .name(PACKET_TYPE) + .help("Send packets to client of instance " + destination) + .create(); + this.emptyBatchesCounter = Counter.build() + .labelNames(DEST_LABELS) + .name(EMPTY_BATCHES) + .help("Send empty batches to client of instance " + destination) + .create(); + this.errorsCounter = Counter.build() + .labelNames(new String[]{DESTINATION, "errorCode"}) + .name(ERRORS) + .help("Client request errors of instance " + destination) + .create(); + this.responseLatency = Histogram.build() + .labelNames(DEST_LABELS) + .name(LATENCY) + .help("Client request latency of instance " + destination) + // buckets in milliseconds + .buckets(1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0) + .create(); + } + + @Override + public String getDestination() { + return this.destination; + } + + @Override + public void profiling(ClientRequestResult result) { + PacketType type = result.getType(); + outboundCounter.labels(destLabelValues).inc(result.getAmount()); + packetsCounter.labels(destination, type.name()).inc(); + short errorCode = result.getErrorCode(); + if (errorCode > 0) { + errorsCounter.labels(destination, Short.toString(errorCode)).inc(); + } + long latency = result.getLatency(); + responseLatency.labels(destLabelValues).observe(latency / 1000000); + switch (type) { + case GET: + boolean empty = result.getEmpty(); + if (empty) { + emptyBatchesCounter.labels(destLabelValues).inc(); + } + break; + // reserve for others + default: + break; + } + } + + @Override public void start() { + if (outboundCounter != null) { + outboundCounter.register(); + } + if (packetsCounter != null) { + packetsCounter.register(); + } + if (emptyBatchesCounter != null) { + emptyBatchesCounter.register(); + } + if (errorsCounter != null) { + errorsCounter.register(); + } + if (responseLatency != null) { + responseLatency.register(); + } + running = true; + } + + @Override public void stop() { + running = false; + if (outboundCounter != null) { + CollectorRegistry.defaultRegistry.unregister(outboundCounter); + } + if (packetsCounter != null) { + CollectorRegistry.defaultRegistry.unregister(packetsCounter); + } + if (emptyBatchesCounter != null) { + CollectorRegistry.defaultRegistry.unregister(emptyBatchesCounter); + } + if (errorsCounter != null) { + CollectorRegistry.defaultRegistry.unregister(errorsCounter); + } + if (responseLatency != null) { + CollectorRegistry.defaultRegistry.unregister(responseLatency); + } + } + + @Override public boolean isStart() { + return running; + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java new file mode 100644 index 00000000..6f80c221 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java @@ -0,0 +1,15 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.server.netty.ClientInstanceProfiler; +import com.alibaba.otter.canal.server.netty.ClientInstanceProfilerFactory; + +/** + * @author Chuanyi Li + */ +public class PrometheusClientInstanceProfilerFactory implements ClientInstanceProfilerFactory { + + @Override + public ClientInstanceProfiler create(String destination) { + return new PrometheusClientInstanceProfiler(destination); + } +} diff --git a/prometheus/src/main/resources/META-INF/aop.xml b/prometheus/src/main/resources/META-INF/aop.xml deleted file mode 100644 index 8bd31d0c..00000000 --- a/prometheus/src/main/resources/META-INF/aop.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java index 9e9d9a39..cfd05ee6 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java @@ -10,7 +10,7 @@ import java.util.concurrent.ConcurrentMap; */ public class CanalServerWithNettyProfiler { - private static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); + public static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); private volatile ClientInstanceProfilerFactory factory; private final ConcurrentMap cliPfs; private final CanalServerWithEmbedded server; @@ -29,34 +29,47 @@ public class CanalServerWithNettyProfiler { return SingletonHolder.SINGLETON; } - public void profiling(String dest, ClientRequestResult result) { + public void profiling(String destination, ClientRequestResult result) { if (isDisabled()) { return; } - ClientInstanceProfiler profiler = tryGet(dest); + ClientInstanceProfiler profiler = cliPfs.get(destination); if (profiler != null) { profiler.profiling(result); } } - /** - * Remove instance profiler for specified instance. - * Only accepted while instance is not running. - * @param dest canal instance destination - */ - public void remove(String dest) throws IllegalStateException { + public void start(String destination) { if (isDisabled()) { return; } - synchronized (cliPfs) { - if (server.isStart(dest)) { - throw new IllegalStateException("Instance profiler should not be removed while running."); - } - cliPfs.remove(dest); + if (server.isStart(destination)) { + throw new IllegalStateException("Instance profiler should not be start while running."); + } + ClientInstanceProfiler profiler = factory.create(destination); + profiler.start(); + cliPfs.put(destination, profiler); + } + + /** + * Remove instance profiler for specified instance. + * Only accepted while instance is not running. + * @param destination canal instance destination + */ + public void stop(String destination) { + if (isDisabled()) { + return; + } + if (server.isStart(destination)) { + throw new IllegalStateException("Instance profiler should not be stop while running."); + } + ClientInstanceProfiler profiler = cliPfs.remove(destination); + if (profiler != null && profiler.isStart()) { + profiler.stop(); } } - public void setFactory(ClientInstanceProfilerFactory factory) { + public void setInstanceProfilerFactory(ClientInstanceProfilerFactory factory) { this.factory = factory; } @@ -64,15 +77,18 @@ public class CanalServerWithNettyProfiler { return factory == DISABLED || factory == null; } - private ClientInstanceProfiler tryGet(String dest) { + private ClientInstanceProfiler tryGet(String destination) { //try fast get - ClientInstanceProfiler profiler = cliPfs.get(dest); + ClientInstanceProfiler profiler = cliPfs.get(destination); if (profiler == null) { synchronized (cliPfs) { - if (server.isStart(dest)) { + if (server.isStart(destination)) { // avoid overwriting - cliPfs.putIfAbsent(dest, factory.create()); - profiler = cliPfs.get(dest); + cliPfs.putIfAbsent(destination, factory.create(destination)); + profiler = cliPfs.get(destination); + if (!profiler.isStart()) { + profiler.start(); + } } } } @@ -81,7 +97,7 @@ public class CanalServerWithNettyProfiler { private static class DefaultClientInstanceProfilerFactory implements ClientInstanceProfilerFactory { @Override - public ClientInstanceProfiler create() { + public ClientInstanceProfiler create(String destination) { throw new UnsupportedOperationException(); } } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java index 6c9ffc8b..bd4e2dbc 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java @@ -1,11 +1,12 @@ package com.alibaba.otter.canal.server.netty; +import com.alibaba.otter.canal.common.CanalLifeCycle; import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator; /** * @author Chuanyi Li */ -public interface ClientInstanceProfiler { +public interface ClientInstanceProfiler extends CanalLifeCycle { String getDestination(); diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java index 30075ba1..12b54bc6 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java @@ -5,6 +5,6 @@ package com.alibaba.otter.canal.server.netty; */ public interface ClientInstanceProfilerFactory { - ClientInstanceProfiler create(); + ClientInstanceProfiler create(String destination); } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java index 7ea861d4..2d09f87e 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java @@ -3,6 +3,7 @@ package com.alibaba.otter.canal.server.netty.handler; import java.util.List; import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.jboss.netty.buffer.ChannelBuffer; diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java index 37ee88e2..19a33667 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java @@ -15,18 +15,27 @@ public class ChannelFutureAggregator implements ChannelFutureListener { private ClientRequestResult result; - public ChannelFutureAggregator(String dest, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency) { - this(dest, request, type, amount, latency, 0); + public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, boolean empty) { + this(destination, request, type, amount, latency, empty, (short) 0); } - private ChannelFutureAggregator(String dest, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, int errorCode) { + public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency) { + this(destination, request, type, amount, latency, false, (short) 0); + } + + public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, short errorCode) { + this(destination, request, type, amount, latency, false, errorCode); + } + + private ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, boolean empty, short errorCode) { this.result = new ClientRequestResult.Builder() - .dest(dest) + .destination(destination) .type(type) .request(request) .amount(amount) .latency(latency) .errorCode(errorCode) + .empty(empty) .build(); } @@ -36,7 +45,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener { if (future.getCause() != null) { result.channelError = future.getCause(); } - profiler().profiling(result.dest, result); + profiler().profiling(result.destination, result); } /** @@ -44,39 +53,42 @@ public class ChannelFutureAggregator implements ChannelFutureListener { */ public static class ClientRequestResult { - private String dest; + private String destination; private CanalPacket.PacketType type; private GeneratedMessage request; - private int amount; - private long latency; - private int errorCode; - private Throwable channelError; + private int amount; + private long latency; + private short errorCode; + private boolean empty; + private Throwable channelError; private ClientRequestResult() {} private ClientRequestResult(Builder builder) { - this.dest = Preconditions.checkNotNull(builder.dest); + this.destination = Preconditions.checkNotNull(builder.destination); this.type = Preconditions.checkNotNull(builder.type); this.request = Preconditions.checkNotNull(builder.request); this.amount = builder.amount; this.latency = builder.latency; this.errorCode = builder.errorCode; + this.empty = builder.empty; this.channelError = builder.channelError; } // auto-generated public static class Builder { - private String dest; + private String destination; private CanalPacket.PacketType type; private GeneratedMessage request; private int amount; private long latency; - private int errorCode; + private short errorCode; + private boolean empty; private Throwable channelError; - Builder dest(String dest) { - this.dest = dest; + Builder destination(String destination) { + this.destination = destination; return this; } @@ -100,23 +112,29 @@ public class ChannelFutureAggregator implements ChannelFutureListener { return this; } - Builder errorCode(int errorCode) { + Builder errorCode(short errorCode) { this.errorCode = errorCode; return this; } + Builder empty(boolean empty) { + this.empty = empty; + return this; + } + public Builder channelError(Throwable channelError) { this.channelError = channelError; return this; } public Builder fromPrototype(ClientRequestResult prototype) { - dest = prototype.dest; + destination = prototype.destination; type = prototype.type; request = prototype.request; amount = prototype.amount; latency = prototype.latency; errorCode = prototype.errorCode; + empty = prototype.empty; channelError = prototype.channelError; return this; } @@ -126,8 +144,8 @@ public class ChannelFutureAggregator implements ChannelFutureListener { } } // getters - public String getDest() { - return dest; + public String getDestination() { + return destination; } public CanalPacket.PacketType getType() { @@ -146,10 +164,14 @@ public class ChannelFutureAggregator implements ChannelFutureListener { return latency; } - public int getErrorCode() { + public short getErrorCode() { return errorCode; } + public boolean getEmpty() { + return empty; + } + public Throwable getChannelError() { return channelError; } diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java index 6511e6ba..23c5071b 100644 --- a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java +++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java @@ -39,6 +39,7 @@ public class EntryEventSink extends AbstractCanalEventSink> handler : getHandlers()) { events = handler.before(events); } - + long blockingStart = 0L; int fullTimes = 0; do { if (eventStore.tryPut(events)) { + if (fullTimes > 0) { + eventsSinkBlockingTime.addAndGet(System.nanoTime() - blockingStart); + } for (CanalEventDownStreamHandler> handler : getHandlers()) { events = handler.after(events); } return true; } else { + if (fullTimes == 0) { + blockingStart = System.nanoTime(); + } applyWait(++fullTimes); } @@ -202,4 +209,8 @@ public class EntryEventSink extends AbstractCanalEventSink Date: Mon, 6 Aug 2018 11:11:57 +0800 Subject: [PATCH 12/21] fix bug: kafka get row data for performance --- .../alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java index e88f9354..0eedfa7a 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java @@ -121,7 +121,7 @@ public class CanalKafkaStarter { Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据 long batchId = message.getId(); try { - int size = message.getEntries().size(); + int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); if (batchId != -1 && size != 0) { if (!StringUtils.isEmpty(destination.getTopic())) { Topic topic = new Topic(); From 7662c9241e19ab26874a781b8296524c3a8ed480 Mon Sep 17 00:00:00 2001 From: winger Date: Fri, 3 Aug 2018 11:11:38 +0800 Subject: [PATCH 13/21] =?UTF-8?q?=E6=94=AF=E6=8C=81GTID=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E9=80=9A=E8=BF=87=E8=8E=B7=E5=8F=96=E5=BD=93?= =?UTF-8?q?=E5=89=8Dgtid=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/parse/inbound/mysql/MysqlEventParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 6c2b9d4d..e640364f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -643,6 +643,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation"); } EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1))); + if (isGTIDMode && fields.size() > 4) { + endPosition.setGtid(fields.get(4)); + } return endPosition; } catch (IOException e) { throw new CanalParseException("command : 'show master status' has an error!", e); From ffeec8fc86938b6fd3a7aa73135fc927f126bd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Fri, 3 Aug 2018 17:48:32 +0800 Subject: [PATCH 14/21] fixed issue #801 , support canal docker --- .gitignore | 2 + .../otter/canal/deployer/CanalController.java | 13 +- deployer/src/main/resources/canal.properties | 9 +- .../resources/example/instance.properties | 19 ++- .../resources/spring/default-instance.xml | 4 +- .../main/resources/spring/file-instance.xml | 4 +- .../main/resources/spring/group-instance.xml | 8 +- .../main/resources/spring/local-instance.xml | 2 +- .../main/resources/spring/memory-instance.xml | 4 +- docker/Dockerfile | 69 +++++++++++ docker/build.sh | 30 +++++ docker/image/admin/app.sh | 117 ++++++++++++++++++ docker/image/admin/bin/clean_log | 2 + docker/image/admin/bin/clean_log.sh | 45 +++++++ docker/image/admin/health.sh | 13 ++ docker/image/alidata/bin/exec_rc_local.sh | 11 ++ docker/image/alidata/bin/lark-wait | 6 + docker/image/alidata/bin/main.sh | 27 ++++ docker/image/alidata/init/02init-sshd.sh | 19 +++ docker/image/alidata/init/fix-hosts.py | 66 ++++++++++ docker/image/alidata/lib/proc.sh | 40 ++++++ docker/run.sh | 92 ++++++++++++++ .../parse/driver/mysql/MysqlConnector.java | 2 +- 23 files changed, 580 insertions(+), 24 deletions(-) create mode 100644 docker/Dockerfile create mode 100644 docker/build.sh create mode 100755 docker/image/admin/app.sh create mode 100644 docker/image/admin/bin/clean_log create mode 100755 docker/image/admin/bin/clean_log.sh create mode 100755 docker/image/admin/health.sh create mode 100755 docker/image/alidata/bin/exec_rc_local.sh create mode 100755 docker/image/alidata/bin/lark-wait create mode 100755 docker/image/alidata/bin/main.sh create mode 100755 docker/image/alidata/init/02init-sshd.sh create mode 100755 docker/image/alidata/init/fix-hosts.py create mode 100644 docker/image/alidata/lib/proc.sh create mode 100644 docker/run.sh diff --git a/.gitignore b/.gitignore index 3f1a5d6b..90bd720f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ jtester.properties .idea/ *.iml .DS_Store +*.tar.gz +*.rpm diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java index 88f6e9a3..0a2c91f8 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java @@ -379,7 +379,18 @@ public class CanalController { } private String getProperty(Properties properties, String key) { - return StringUtils.trim(properties.getProperty(StringUtils.trim(key))); + key = StringUtils.trim(key); + String value = System.getProperty(key); + + if (value == null) { + value = System.getenv(key); + } + + if (value == null) { + value = properties.getProperty(key); + } + + return StringUtils.trim(value); } public void start() throws Throwable { diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 0a11a5c9..2bff29ae 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -3,7 +3,7 @@ ################################################# canal.id= 1 canal.ip= -canal.port= 11111 +canal.port=11111 canal.zkServers= # flush data to zk canal.zookeeper.flush.period = 1000 @@ -59,6 +59,13 @@ canal.instance.parser.parallel = true ## disruptor ringbuffer size, must be power of 2 canal.instance.parser.parallelBufferSize = 256 +# table meta tsdb info +canal.instance.tsdb.enable=true +canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +canal.instance.tsdb.dbUsername=canal +canal.instance.tsdb.dbPassword=canal + ################################################# ######### destinations ############# ################################################# diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index a3703b86..e0cf0064 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -1,11 +1,12 @@ ################################################# -## mysql serverId -canal.instance.mysql.slaveId=0 +## mysql serverId , v1.0.26+ will autoGen +# canal.instance.mysql.slaveId=0 + +# enable gtid use true/false +canal.instance.gtidon=false # position info canal.instance.master.address=127.0.0.1:3306 -# enable gtid use true/false -canal.instance.gtidon=false canal.instance.master.journal.name= canal.instance.master.position= canal.instance.master.timestamp= @@ -13,23 +14,21 @@ canal.instance.master.gtid= # table meta tsdb info canal.instance.tsdb.enable=true -canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} -canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb -canal.instance.tsdb.dbUsername=canal -canal.instance.tsdb.dbPassword=canal - +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal #canal.instance.standby.address = #canal.instance.standby.journal.name = #canal.instance.standby.position = #canal.instance.standby.timestamp = #canal.instance.standby.gtid= + # username/password canal.instance.dbUsername=canal canal.instance.dbPassword=canal -canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 + # table regex canal.instance.filter.regex=.*\\..* # table black regex diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index 9bf28896..6b455c83 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -149,7 +149,7 @@ - + @@ -157,7 +157,7 @@ - + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 6d263717..377428a9 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -134,7 +134,7 @@ - + @@ -142,7 +142,7 @@ - + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 60ceff01..44f1ab3b 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -131,7 +131,7 @@ - + @@ -139,7 +139,7 @@ - + @@ -229,7 +229,7 @@ - + @@ -237,7 +237,7 @@ - + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index 2020d0d6..930ecb7b 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -114,7 +114,7 @@ - + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index c08f3f12..cd387b8f 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -122,7 +122,7 @@ - + @@ -130,7 +130,7 @@ - + diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..7e10085b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,69 @@ +FROM centos:centos6.7 + +MAINTAINER agapple (jianghang115@gmail.com) + +# install system +RUN \ + /bin/cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo 'root:Hello1234' | chpasswd && \ + groupadd -r admin && useradd -g admin admin && \ + yum install -y man && \ + yum install -y dstat && \ + yum install -y unzip && \ + yum install -y nc && \ + yum install -y openssh-server && \ + yum install -y tar && \ + yum install -y which && \ + yum install -y wget && \ + yum install -y perl && \ + yum install -y file && \ + ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \ + ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \ + sed -ri 's/session required pam_loginuid.so/#session required pam_loginuid.so/g' /etc/pam.d/sshd && \ + sed -i -e 's/^#Port 22$/Port 2222/' /etc/ssh/sshd_config && \ + mkdir -p /root/.ssh && chown root.root /root && chmod 700 /root/.ssh && \ + yum install -y cronie && \ + sed -i '/session required pam_loginuid.so/d' /etc/pam.d/crond && \ + yum clean all && \ + true + +# install canal +COPY image/ /tmp/docker/ +COPY canal.deployer-*.tar.gz /home/admin/ +COPY jdk-8-linux-x64.rpm /tmp/ + +RUN \ + cp -R /tmp/docker/alidata /alidata && \ + chmod +x /alidata/bin/* && \ + mkdir -p /home/admin && \ + cp -R /tmp/docker/admin/* /home/admin/ && \ + /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \ + + touch /var/lib/rpm/* && \ + yum -y install /tmp/jdk-8-linux-x64.rpm && \ + /bin/rm -f /tmp/jdk-8-linux-x64.rpm && \ + + echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ + echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ + /bin/mv /home/admin/bin/canal-check /etc/cron.d && \ + /bin/mv /home/admin/bin/clean_log /etc/cron.d && \ + chmod +x /home/admin/bin/canal-process-check.py && \ + + mkdir -p /home/admin/canal-server && \ + tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ + /bin/rm -f /home/admin/canal.deployer-*.tar.gz && \ + + mkdir -p home/admin/canal-server/logs && \ + chmod +x /home/admin/*.sh && \ + chmod +x /home/admin/bin/*.sh && \ + chown admin: -R /home/admin && \ + yum clean all && \ + true + +# 2222 sys , 8080 web , 8000 debug , 11111 canal +EXPOSE 2222 11111 8000 8080 + +WORKDIR /home/admin + +ENTRYPOINT [ "/alidata/bin/main.sh" ] +CMD [ "/home/admin/app.sh" ] \ No newline at end of file diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 00000000..58424fb3 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} + +if [ ! -f $BASE/jdk*.rpm ] ; then + DOWNLOAD_LINK="http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz" + wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=xxx; oraclelicense=accept-securebackup-cookie" "$DOWNLOAD_LINK" -O $BASE/jdk-8-linux-x64.rpm +fi + +cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ; + +if [ "$1" == "kafka" ] ; then + cp $BASE/../target/canal-kafka-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-server $BASE/ +else + cp $BASE/../target/canal.deployer-*.tar.gz $BASE/ + docker build --no-cache -t canal/canal-server $BASE/ +fi diff --git a/docker/image/admin/app.sh b/docker/image/admin/app.sh new file mode 100755 index 00000000..254f7e48 --- /dev/null +++ b/docker/image/admin/app.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e + +source /etc/profile +export JAVA_HOME=/usr/java/latest +export PATH=$JAVA_HOME/bin:$PATH +touch /tmp/start.log +chown admin: /tmp/start.log +chown -R admin: /home/admin/canal-server +host=`hostname -i` + +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} + + +function checkStart() { + local name=$1 + local cmd=$2 + local timeout=$3 + cost=5 + while [ $timeout -gt 0 ]; do + ST=`eval $cmd` + if [ "$ST" == "0" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + elif [ "$ST" == "" ]; then + sleep 1 + let timeout=timeout-1 + let cost=cost+1 + else + break + fi + done + echo "start $name successful" +} + + +function start_canal() { + echo "start canal ..." + serverPort=`perl -le 'print $ENV{"canal.port"}'` + if [ -z "$serverPort" ] ; then + serverPort=11111 + fi + + destination=`perl -le 'print $ENV{"canal.destinations"}'` + if [[ "$destination" =~ ',' ]]; then + echo "multi destination:$destination is not support" + exit 1; + else + mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination + fi + su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1' + sleep 5 + #check start + checkStart "canal" "nc 127.0.0.1 $serverPort -w 1 -z | wc -l" 30 +} + +function stop_canal() { + echo "stop canal" + su admin -c 'cd /home/admin/canal-server/bin/ && sh stop.sh 1>>/tmp/start.log 2>&1' + echo "stop canal successful ..." +} + +echo "==> START ..." + +start_canal + +echo "==> START SUCCESSFUL ..." + +tail -f /dev/null & +# wait TERM signal +waitterm + +echo "==> STOP" + +stop_canal + +echo "==> STOP SUCCESSFUL ..." diff --git a/docker/image/admin/bin/clean_log b/docker/image/admin/bin/clean_log new file mode 100644 index 00000000..df183614 --- /dev/null +++ b/docker/image/admin/bin/clean_log @@ -0,0 +1,2 @@ +# cron clean log once per minute +*/2 * * * * admin /home/admin/bin/clean_log.sh >>/tmp/clean_log.log 2>&1 diff --git a/docker/image/admin/bin/clean_log.sh b/docker/image/admin/bin/clean_log.sh new file mode 100755 index 00000000..9d0a0bef --- /dev/null +++ b/docker/image/admin/bin/clean_log.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Global Settings +PATH="$HOME/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/X11R6/bin:/root/bin" +export PATH + +CUTOFF="85" +#获取磁盘使用率最高的分区 +USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) +before=$USAGE + +baseClean(){ + #删除tmp目录15天前的文件。 + #更新文档时间戳 + if [ -d /tmp/hsperfdata_admin ] + then + touch /tmp/hsperfdata_admin + touch /tmp/hsperfdata_admin/* + fi + + find /tmp/ -type f -mtime +15 | xargs -t rm -rf >/dev/null 2>&1 + + + now=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + echo "before:$before; now:$now" +} + +CANAL_DIR="/home/admin/canal-server/logs" +if [[ -d $CANAL_DIR ]]; then + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 90 ]]; then + find $CANAL_DIR -type f -mtime +7 | xargs rm -rf {} + fi + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 80 ]]; then + find $CANAL_DIR -type f -mtime +3 | xargs rm -rf {} + fi + USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) + if [[ $USAGE -ge 80 ]]; then + find $CANAL_DIR -type d -empty -mtime +3 | grep -v canal | xargs rm -rf {} + find $CANAL_DIR -type f -iname '*.tmp' | xargs rm -rf {} + fi + baseClean + exit 0 +fi \ No newline at end of file diff --git a/docker/image/admin/health.sh b/docker/image/admin/health.sh new file mode 100755 index 00000000..17a89740 --- /dev/null +++ b/docker/image/admin/health.sh @@ -0,0 +1,13 @@ +#!/bin/sh +CHECK_URL="http://127.0.0.1:8080/metrics" +CHECK_POINT="success" +CHECK_COUNT=`curl -s --connect-timeout 7 --max-time 7 $CHECK_URL | grep -c $CHECK_POINT` +if [ $CHECK_COUNT -eq 0 ]; then + echo "[FAILED]" + status=0 + error=1 +else + echo "[ OK ]" + status=1 + error=0 +fi \ No newline at end of file diff --git a/docker/image/alidata/bin/exec_rc_local.sh b/docker/image/alidata/bin/exec_rc_local.sh new file mode 100755 index 00000000..7cc8aa3f --- /dev/null +++ b/docker/image/alidata/bin/exec_rc_local.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ "${SKIP_EXEC_RC_LOCAL}" = "YES" ] ; then + echo "skip /etc/rc.local: SKIP_EXEC_RC_LOCAL=${SKIP_EXEC_RC_LOCAL}" + exit +fi + +if [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] ; then + echo "skip /etc/rc.local: DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" + exit +fi \ No newline at end of file diff --git a/docker/image/alidata/bin/lark-wait b/docker/image/alidata/bin/lark-wait new file mode 100755 index 00000000..db7be708 --- /dev/null +++ b/docker/image/alidata/bin/lark-wait @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +chown admin: -R /home/admin/ +source /alidata/lib/proc.sh +waitterm diff --git a/docker/image/alidata/bin/main.sh b/docker/image/alidata/bin/main.sh new file mode 100755 index 00000000..a57551a7 --- /dev/null +++ b/docker/image/alidata/bin/main.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +[ -n "${DOCKER_DEPLOY_TYPE}" ] || DOCKER_DEPLOY_TYPE="VM" +echo "DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" + +# run init scripts +for e in $(ls /alidata/init/*) ; do + [ -x "${e}" ] || continue + echo "==> INIT $e" + $e + echo "==> EXIT CODE: $?" +done + +echo "==> INIT DEFAULT" +service sshd start +service crond start + +#echo "check hostname -i: `hostname -i`" +#hti_num=`hostname -i|awk '{print NF}'` +#if [ $hti_num -gt 1 ];then +# echo "hostname -i result error:`hostname -i`" +# exit 120 +#fi + +echo "==> INIT DONE" +echo "==> RUN ${*}" +exec "${@}" \ No newline at end of file diff --git a/docker/image/alidata/init/02init-sshd.sh b/docker/image/alidata/init/02init-sshd.sh new file mode 100755 index 00000000..118ee553 --- /dev/null +++ b/docker/image/alidata/init/02init-sshd.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# set port +if [ -z "${SSHD_PORT}" ] ; then + SSHD_PORT=22 + [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] && SSHD_PORT=2222 +fi + +sed -r -i '/^OPTIONS=/ d' /etc/sysconfig/sshd +echo 'OPTIONS="-p '"${SSHD_PORT}"'"' >> /etc/sysconfig/sshd + +# set admin ssh pulic key +if [ "${USE_ADMIN_PASSAGE}" = "YES" ] ; then + echo "set admin passage" + mkdir -p /home/admin/.ssh + chown admin:admin /home/admin/.ssh + chown admin:admin /home/admin/.ssh/authorized_keys + chmod 644 /home/admin/.ssh/authorized_keys +fi diff --git a/docker/image/alidata/init/fix-hosts.py b/docker/image/alidata/init/fix-hosts.py new file mode 100755 index 00000000..4ee8abb5 --- /dev/null +++ b/docker/image/alidata/init/fix-hosts.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +#****************************************************************# +# Create Date: 2017-01-06 17:58 +#***************************************************************# + +import socket +import shutil +from time import gmtime, strftime + +# get host_name +host_name = socket.gethostname() +tmp_file = "/tmp/.lark-fix-host.hosts" +host_file = "/etc/hosts" +bak_file_name = "/tmp/hosts-fix-bak.%s" % ( strftime("%Y-%m-%d_%H-%M-%S", gmtime()) ) + +# load /etc/hosts file context +FH = open(host_file,"r") +file_lines = [ i.rstrip() for i in FH.readlines()] +FH.close() +file_lines_reverse = file_lines[::-1] +new_lines = [] +bad_lines = [] +last_match_line = "" + +for line in file_lines_reverse: + if line.find(host_name) < 0: # 不匹配的行直接跳过 + new_lines.append(line + "\n") + continue + + cols = line.split() + new_cols = [] + if cols[0].startswith("#"): # 跳过已经注释掉的行 + new_lines.append(line + "\n") + continue + for col in cols: + if not col == host_name: # 跳过不匹配的列 + new_cols.append(col) + continue + + if cols[0] == "127.0.0.1": # 如果第一列是 127.0.0.1 就跳过匹配的列, 防止 hostname -i 返回 127.0.0.1 + continue + + # 如果已经发现过匹配的列, 就丢掉重复的列 + if not len(last_match_line) == 0: + continue + + new_cols.append(col) + last_match_line = line + + # 跳过 xx.xx.xx.xx hostname 这样的重复列 + if len(new_cols) == 1: + continue + + new_l = "%s\n" % " ".join(new_cols) + new_lines.append(new_l) + +# save tmp hosts + +FH2=file(tmp_file,"w+") +FH2.writelines( new_lines[::-1]) +FH2.close() + +# mv to /etc/hosts +shutil.copy(host_file, bak_file_name) +shutil.move(tmp_file, host_file) diff --git a/docker/image/alidata/lib/proc.sh b/docker/image/alidata/lib/proc.sh new file mode 100644 index 00000000..40ddff93 --- /dev/null +++ b/docker/image/alidata/lib/proc.sh @@ -0,0 +1,40 @@ +# waitterm +# wait TERM/INT signal. +# see: http://veithen.github.io/2014/11/16/sigterm-propagation.html +waitterm() { + local PID + # any process to block + tail -f /dev/null & + PID="$!" + # setup trap, could do nothing, or just kill the blocker + trap "kill -TERM ${PID}" TERM INT + # wait for signal, ignore wait exit code + wait "${PID}" || true + # clear trap + trap - TERM INT + # wait blocker, ignore blocker exit code + wait "${PID}" 2>/dev/null || true +} + +# waittermpid "${PIDFILE}". +# monitor process by pidfile && wait TERM/INT signal. +# if the process disappeared, return 1, means exit with ERROR. +# if TERM or INT signal received, return 0, means OK to exit. +waittermpid() { + local PIDFILE PID do_run error + PIDFILE="${1?}" + do_run=true + error=0 + trap "do_run=false" TERM INT + while "${do_run}" ; do + PID="$(cat "${PIDFILE}")" + if ! ps -p "${PID}" >/dev/null 2>&1 ; then + do_run=false + error=1 + else + sleep 1 + fi + done + trap - TERM INT + return "${error}" +} diff --git a/docker/run.sh b/docker/run.sh new file mode 100644 index 00000000..7c56e2ba --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +function usage() { + echo "Usage:" + echo " run.sh [CONFIG]" + echo "example:" + echo " run.sh -e canal.instance.master.address=127.0.0.1:3306 \\" + echo " -e canal.instance.dbUsername=canal \\" + echo " -e canal.instance.dbPassword=canal \\" + echo " -e canal.instance.connectionCharset=UTF-8 \\" + echo " -e canal.instance.tsdb.enable=true \\" + echo " -e canal.instance.gtidon=false \\" + echo " -e canal.instance.filter.regex=.*\\..* " + exit +} + +function check_port() { + local port=$1 + local TL=$(which telnet) + if [ -f $TL ]; then + data=`echo quit | telnet 127.0.0.1 $port| grep -ic connected` + echo $data + return + fi + + local NC=$(which nc) + if [ -f $NC ]; then + data=`nc -z -w 1 127.0.0.1 $port | grep -ic succeeded` + echo $data + return + fi + echo "0" + return +} + +function getMyIp() { + case "`uname`" in + Darwin) + myip=`echo "show State:/Network/Global/IPv4" | scutil | grep PrimaryInterface | awk '{print $3}' | xargs ifconfig | grep inet | grep -v inet6 | awk '{print $2}'` + ;; + *) + myip=`ip route get 1 | awk '{print $NF;exit}'` + ;; + esac + echo $myip +} + +NET_MODE="" +case "`uname`" in + Darwin) + bin_abs_path=`cd $(dirname $0); pwd` + ;; + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + NET_MODE="--net=host" + ;; + *) + NET_MODE="--net=host" + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +BASE=${bin_abs_path} +if [ $# -eq 0 ]; then + usage +elif [ "$1" == "-h" ] ; then + usage +elif [ "$1" == "help" ] ; then + usage +fi + +DATA="$BASE/data" +mkdir -p $DATA +CONFIG=${@:1} +#VOLUMNS="-v $DATA:/home/admin/canal-server/logs" +PORTLIST="8000 8080 2222 11111" +PORTS="" +for PORT in $PORTLIST ; do + #exist=`check_port $PORT` + exist="0" + if [ "$exist" == "0" ]; then + PORTS="$PORTS -p $PORT:$PORT" + else + echo "port $PORT is used , pls check" + exit 1 + fi +done + +MEMORY="-m 4096m" +LOCALHOST=`getMyIp` +cmd="docker run -it -h $LOCALHOST $CONFIG --name=canal-server $VOLUMNS $NET_MODE $PORTS $MEMORY canal/canal-server" +echo $cmd +eval $cmd \ No newline at end of file diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 29b96852..0f01cbd5 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -32,7 +32,7 @@ public class MysqlConnector { private String password; private byte charsetNumber = 33; - private String defaultSchema = "retl"; + private String defaultSchema = "test"; private int soTimeout = 30 * 1000; private int connTimeout = 5 * 1000; private int receiveBufferSize = 16 * 1024; From 325fe89aed27fe7d3525e12726c96dd99e07dddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=83=E9=94=8B?= Date: Sat, 4 Aug 2018 10:37:55 +0800 Subject: [PATCH 15/21] fixed issue #801 , optimizer docker --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e10085b..e44796cb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,9 +45,7 @@ RUN \ echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \ echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \ - /bin/mv /home/admin/bin/canal-check /etc/cron.d && \ /bin/mv /home/admin/bin/clean_log /etc/cron.d && \ - chmod +x /home/admin/bin/canal-process-check.py && \ mkdir -p /home/admin/canal-server && \ tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ From 9786faba9ad54e0ede00eb86642bc158906f2a69 Mon Sep 17 00:00:00 2001 From: qmzhang <578967933@qq.com> Date: Sun, 5 Aug 2018 19:40:14 +0800 Subject: [PATCH 16/21] =?UTF-8?q?instance=E7=94=B1manager=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E6=97=B6=EF=BC=8CCanalParameter#MetaMode=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E6=9C=AC=E5=9C=B0meta.dat=E5=AD=98=E5=82=A8=E4=BD=8D?= =?UTF-8?q?=E7=82=B9=E4=BF=A1=E6=81=AF=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/CanalInstanceWithManager.java | 21 +++----- .../manager/model/CanalParameter.java | 26 ++++++++- .../CanalServerWithEmbedded_FileModeTest.java | 53 +++++++++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java index bca5ba13..35e3235a 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import com.alibaba.otter.canal.meta.FileMixedMetaManager; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,13 +21,7 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.instance.core.AbstractCanalInstance; import com.alibaba.otter.canal.instance.manager.model.Canal; import com.alibaba.otter.canal.instance.manager.model.CanalParameter; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode; -import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; import com.alibaba.otter.canal.meta.MemoryMetaManager; import com.alibaba.otter.canal.meta.PeriodMixedMetaManager; import com.alibaba.otter.canal.meta.ZooKeeperMetaManager; @@ -37,12 +32,7 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; -import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager; -import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager; -import com.alibaba.otter.canal.parse.index.MetaLogPositionManager; -import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager; -import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager; +import com.alibaba.otter.canal.parse.index.*; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.sink.entry.EntryEventSink; @@ -120,6 +110,11 @@ public class CanalInstanceWithManager extends AbstractCanalInstance { ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(getZkclientx()); ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager); + } else if (mode.isLocalFile()){ + FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager(); + fileMixedMetaManager.setDataDir(parameters.getDataDir()); + fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod()); + metaManager = fileMixedMetaManager; } else { throw new CanalException("unsupport MetaMode for " + mode); } diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java index 16c929ca..befc8e2f 100644 --- a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java +++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java @@ -28,8 +28,10 @@ public class CanalParameter implements Serializable { private Long zkClusterId; // zk集群id，为管理方便 private List zkClusters; // zk集群地址 + private String dataDir = "../conf"; // 默认本地文件数据的目录默认是conf // meta相关参数 private MetaMode metaMode = MetaMode.MEMORY; // meta机制 + private Integer metaFileFlushPeriod = 1000; // meta刷新间隔 // storage存储 private Integer transactionSize = 1024; // 支持处理的transaction事务大小 @@ -243,7 +245,9 @@ public class CanalParameter implements Serializable { /** 文件存储模式 */ ZOOKEEPER, /** 混合模式，内存+文件 */ - MIXED; + MIXED, + /** 本地文件存储模式*/ + LOCAL_FILE; public boolean isMemory() { return this.equals(MetaMode.MEMORY); @@ -256,6 +260,10 @@ public class CanalParameter implements Serializable { public boolean isMixed() { return this.equals(MetaMode.MIXED); } + + public boolean isLocalFile(){ + return this.equals(MetaMode.LOCAL_FILE); + } } public static enum IndexMode { @@ -390,6 +398,22 @@ public class CanalParameter implements Serializable { return storageMode; } + public String getDataDir() { + return dataDir; + } + + public void setDataDir(String dataDir) { + this.dataDir = dataDir; + } + + public Integer getMetaFileFlushPeriod() { + return metaFileFlushPeriod; + } + + public void setMetaFileFlushPeriod(Integer metaFileFlushPeriod) { + this.metaFileFlushPeriod = metaFileFlushPeriod; + } + public void setStorageMode(StorageMode storageMode) { this.storageMode = storageMode; } diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java new file mode 100644 index 00000000..e986b6ff --- /dev/null +++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_FileModeTest.java @@ -0,0 +1,53 @@ +package com.alibaba.otter.canal.server; + +import java.net.InetSocketAddress; +import java.util.Arrays; + +import com.alibaba.otter.canal.instance.manager.model.Canal; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.*; + +public class CanalServerWithEmbedded_FileModeTest extends BaseCanalServerWithEmbededTest { + + protected Canal buildCanal() { + Canal canal = new Canal(); + canal.setId(1L); + canal.setName(DESTINATION); + canal.setDesc("my standalone server test "); + + CanalParameter parameter = new CanalParameter(); + + parameter.setMetaMode(MetaMode.LOCAL_FILE); + parameter.setDataDir("./conf"); + parameter.setMetaFileFlushPeriod(1000); + parameter.setHaMode(HAMode.HEARTBEAT); + parameter.setIndexMode(IndexMode.MEMORY_META_FAILBACK); + + parameter.setStorageMode(StorageMode.MEMORY); + parameter.setMemoryStorageBufferSize(32 * 1024); + + parameter.setSourcingType(SourcingType.MYSQL); + parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306), + new InetSocketAddress(MYSQL_ADDRESS, 3306))); + parameter.setDbUsername(USERNAME); + parameter.setDbPassword(PASSWORD); + parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}", + "{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}")); + + parameter.setSlaveId(1234L); + + parameter.setDefaultConnectionTimeoutInSeconds(30); + parameter.setConnectionCharset("UTF-8"); + parameter.setConnectionCharsetNumber((byte) 33); + parameter.setReceiveBufferSize(8 * 1024); + parameter.setSendBufferSize(8 * 1024); + + parameter.setDetectingEnable(false); + parameter.setDetectingIntervalInSeconds(10); + parameter.setDetectingRetryTimes(3); + parameter.setDetectingSQL(DETECTING_SQL); + + canal.setCanalParameter(parameter); + return canal; + } +} \ No newline at end of file From 1cd77bf551665de04381834f646b83d9c1de4179 Mon Sep 17 00:00:00 2001 From: winger Date: Mon, 6 Aug 2018 11:11:57 +0800 Subject: [PATCH 17/21] fix bug: kafka get row data for performance --- .../alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java index e88f9354..0eedfa7a 100644 --- a/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java +++ b/kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaStarter.java @@ -121,7 +121,7 @@ public class CanalKafkaStarter { Message message = server.getWithoutAck(clientIdentity, kafkaProperties.getCanalBatchSize()); // 获取指定数量的数据 long batchId = message.getId(); try { - int size = message.getEntries().size(); + int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size(); if (batchId != -1 && size != 0) { if (!StringUtils.isEmpty(destination.getTopic())) { Topic topic = new Topic(); From 680faea2bbef6a5596dbf20ba28018ae23cc8c96 Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Mon, 6 Aug 2018 17:22:28 +0800 Subject: [PATCH 18/21] Add listener to write. --- .../prometheus/CanalInstanceExports.java | 6 +- .../canal/prometheus/PrometheusService.java | 7 +- .../prometheus/impl/MysqlParserCollector.java | 4 +- .../PrometheusClientInstanceProfiler.java | 6 +- .../netty/CanalServerWithNettyProfiler.java | 12 +-- .../otter/canal/server/netty/NettyUtils.java | 16 ++++ .../handler/ClientAuthenticationHandler.java | 2 +- .../server/netty/handler/SessionHandler.java | 75 +++++++++---------- .../listener/ChannelFutureAggregator.java | 2 +- 9 files changed, 71 insertions(+), 59 deletions(-) diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java index 19759154..388f8e59 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -21,9 +21,9 @@ import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler. public class CanalInstanceExports { private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); - public static final String DESTINATION = "destination"; - public static final String[] DEST_LABELS = {DESTINATION}; - public static final List DEST_LABEL_LIST = Collections.singletonList(DESTINATION); + public static final String DEST = "destination"; + public static final String[] DEST_LABELS = {DEST}; + public static final List DEST_LABEL_LIST = Collections.singletonList(DEST); private final String destination; private Collector storeCollector; private Collector delayCollector; diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java index 2aa7dcd0..5d417883 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -22,11 +22,8 @@ import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler. public class PrometheusService implements CanalMetricsService { private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); - private final Map exports = new ConcurrentHashMap(); - private volatile boolean running = false; - private HTTPServer server; private PrometheusService() { @@ -69,8 +66,8 @@ public class PrometheusService implements CanalMetricsService { // Normally, service should be terminated at canal shutdown. // No need to unregister instance exports explicitly. // But for the sake of safety, unregister them. - for (CanalInstanceExports ie : exports.values()) { - ie.unregister(); + for (CanalInstanceExports cie : exports.values()) { + cie.unregister(); } profiler().setInstanceProfilerFactory(DISABLED); if (server != null) { diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java index 02d35209..c8543f3c 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java @@ -12,7 +12,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DESTINATION; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST; import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; /** @@ -45,7 +45,7 @@ public class MysqlParserCollector extends Collector { this.receivedBinlogBytesHelp = "Received binlog bytes of instance" + destination; this.isParallel = ((MysqlEventParser)parser).isParallel(); this.modeHelp = "Parser mode of instance" + destination; - this.modeLabels = Arrays.asList(DESTINATION, MODE_LABEL); + this.modeLabels = Arrays.asList(DEST, MODE_LABEL); this.modeLabelValues = Arrays.asList(destination, isParallel.toString()); } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java index d3413455..4cbbdb89 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java @@ -7,7 +7,7 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Counter; import io.prometheus.client.Histogram; -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DESTINATION; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST; import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS; /** @@ -38,7 +38,7 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler .help("Send bytes to client of instance " + destination) .create(); this.packetsCounter = Counter.build() - .labelNames(new String[]{DESTINATION, "packetType"}) + .labelNames(new String[]{DEST, "packetType"}) .name(PACKET_TYPE) .help("Send packets to client of instance " + destination) .create(); @@ -48,7 +48,7 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler .help("Send empty batches to client of instance " + destination) .create(); this.errorsCounter = Counter.build() - .labelNames(new String[]{DESTINATION, "errorCode"}) + .labelNames(new String[]{DEST, "errorCode"}) .name(ERRORS) .help("Client request errors of instance " + destination) .create(); diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java index cfd05ee6..e44476d7 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java @@ -11,7 +11,7 @@ import java.util.concurrent.ConcurrentMap; public class CanalServerWithNettyProfiler { public static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); - private volatile ClientInstanceProfilerFactory factory; + private volatile ClientInstanceProfilerFactory clientInstanceProfilerFactory; private final ConcurrentMap cliPfs; private final CanalServerWithEmbedded server; @@ -20,7 +20,7 @@ public class CanalServerWithNettyProfiler { } private CanalServerWithNettyProfiler() { - this.factory = DISABLED; + this.clientInstanceProfilerFactory = DISABLED; this.cliPfs = new ConcurrentHashMap(); this.server = CanalServerWithEmbedded.instance(); } @@ -46,7 +46,7 @@ public class CanalServerWithNettyProfiler { if (server.isStart(destination)) { throw new IllegalStateException("Instance profiler should not be start while running."); } - ClientInstanceProfiler profiler = factory.create(destination); + ClientInstanceProfiler profiler = clientInstanceProfilerFactory.create(destination); profiler.start(); cliPfs.put(destination, profiler); } @@ -70,11 +70,11 @@ public class CanalServerWithNettyProfiler { } public void setInstanceProfilerFactory(ClientInstanceProfilerFactory factory) { - this.factory = factory; + this.clientInstanceProfilerFactory = factory; } private boolean isDisabled() { - return factory == DISABLED || factory == null; + return clientInstanceProfilerFactory == DISABLED || clientInstanceProfilerFactory == null; } private ClientInstanceProfiler tryGet(String destination) { @@ -84,7 +84,7 @@ public class CanalServerWithNettyProfiler { synchronized (cliPfs) { if (server.isStart(destination)) { // avoid overwriting - cliPfs.putIfAbsent(destination, factory.create(destination)); + cliPfs.putIfAbsent(destination, clientInstanceProfilerFactory.create(destination)); profiler = cliPfs.get(destination); if (!profiler.isStart()) { profiler.start(); diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java index 20532400..43ecbe67 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java @@ -74,4 +74,20 @@ public class NettyUtils { .toByteArray(), channelFutureListener); } + + public static byte[] ackPacket() { + return Packet.newBuilder() + .setType(CanalPacket.PacketType.ACK) + .setBody(Ack.newBuilder().build().toByteString()) + .build() + .toByteArray(); + } + + public static byte[] errorPacket(int errorCode, String errorMessage) { + return Packet.newBuilder() + .setType(CanalPacket.PacketType.ACK) + .setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString()) + .build() + .toByteArray(); + } } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java index 60398ea7..597d692a 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java @@ -73,7 +73,7 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler { MDC.remove("destination"); } } - + // 鉴权一次性，暂不统计 NettyUtils.ack(ctx.getChannel(), new ChannelFutureListener() { public void operationComplete(ChannelFuture future) throws Exception { diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java index 2d09f87e..a98e7b8b 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java @@ -52,7 +52,7 @@ public class SessionHandler extends SimpleChannelHandler { public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { logger.info("message receives in session handler..."); - long start = System.currentTimeMillis(); + long start = System.nanoTime(); ChannelBuffer buffer = (ChannelBuffer) e.getMessage(); Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array()); ClientIdentity clientIdentity = null; @@ -76,12 +76,13 @@ public class SessionHandler extends SimpleChannelHandler { embeddedServer.subscribe(clientIdentity); // ctx.setAttachment(clientIdentity);// 设置状态数据 - NettyUtils.ack(ctx.getChannel(), null); + byte[] ackBytes = NettyUtils.ackPacket(); + NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(sub.getDestination(), + sub, packet.getType(), ackBytes.length, System.nanoTime() - start)); } else { - NettyUtils.error(401, - MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage(), - ctx.getChannel(), - null); + byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes ,new ChannelFutureAggregator(sub.getDestination(), + sub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401)); } break; case UNSUBSCRIPTION: @@ -93,12 +94,13 @@ public class SessionHandler extends SimpleChannelHandler { MDC.put("destination", clientIdentity.getDestination()); embeddedServer.unsubscribe(clientIdentity); stopCanalInstanceIfNecessary(clientIdentity);// 尝试关闭 - NettyUtils.ack(ctx.getChannel(), null); + byte[] ackBytes = NettyUtils.ackPacket(); + NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(unsub.getDestination(), + unsub, packet.getType(), ackBytes.length, System.nanoTime() - start)); } else { - NettyUtils.error(401, - MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage(), - ctx.getChannel(), - null); + byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(unsub.getDestination(), + unsub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401)); } break; case GET: @@ -173,7 +175,8 @@ public class SessionHandler extends SimpleChannelHandler { output.writeBytes(2, rowEntries.get(i)); } output.checkNoSpaceLeft(); - NettyUtils.write(ctx.getChannel(), body, null); + NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(), + get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1)); // output.flush(); // byteBuffer.flip(); @@ -194,14 +197,14 @@ public class SessionHandler extends SimpleChannelHandler { } } } - packetBuilder.setBody(messageBuilder.build().toByteString()); - NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据 + byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString()).build().toByteArray(); + NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(), + get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));// 输出数据 } } else { - NettyUtils.error(401, - MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(), - ctx.getChannel(), - null); + byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(get.getDestination(), + get, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401)); } break; case CLIENTACK: @@ -209,10 +212,9 @@ public class SessionHandler extends SimpleChannelHandler { MDC.put("destination", ack.getDestination()); if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) { if (ack.getBatchId() == 0L) { - NettyUtils.error(402, - MessageFormatter.format("batchId should assign value", ack.toString()).getMessage(), - ctx.getChannel(), - null); + byte[] errorBytes = NettyUtils.errorPacket(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(), + ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 402)); } else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据，直接忽略之 // donothing } else { @@ -220,10 +222,9 @@ public class SessionHandler extends SimpleChannelHandler { embeddedServer.ack(clientIdentity, ack.getBatchId()); } } else { - NettyUtils.error(401, - MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage(), - ctx.getChannel(), - null); + byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(), + ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401)); } break; case CLIENTROLLBACK: @@ -239,25 +240,23 @@ public class SessionHandler extends SimpleChannelHandler { embeddedServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次 } } else { - NettyUtils.error(401, - MessageFormatter.format("destination or clientId is null", rollback.toString()) - .getMessage(), - ctx.getChannel(), - null); + byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(rollback.getDestination(), + rollback, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401)); } break; default: - NettyUtils.error(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()) - .getMessage(), ctx.getChannel(), null); + byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(), + null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400)); break; } } catch (Throwable exception) { - NettyUtils.error(400, - MessageFormatter.format("something goes wrong with channel:{}, exception={}", + byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}", ctx.getChannel(), - ExceptionUtils.getStackTrace(exception)).getMessage(), - ctx.getChannel(), - null); + ExceptionUtils.getStackTrace(exception)).getMessage()); + NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(), + null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400)); } finally { MDC.remove("destination"); } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java index 19a33667..89755431 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java @@ -67,7 +67,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener { private ClientRequestResult(Builder builder) { this.destination = Preconditions.checkNotNull(builder.destination); this.type = Preconditions.checkNotNull(builder.type); - this.request = Preconditions.checkNotNull(builder.request); + this.request = builder.request; this.amount = builder.amount; this.latency = builder.latency; this.errorCode = builder.errorCode; From b5a1898cec04be41089483d516d9eca725a0469d Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Tue, 7 Aug 2018 08:18:01 +0800 Subject: [PATCH 19/21] Prometheus refactor. --- .../prometheus/CanalInstanceExports.java | 54 ++++++------ .../canal/prometheus/InstanceRegistry.java | 14 +++ .../canal/prometheus/PrometheusService.java | 31 +++---- .../canal/prometheus/impl/EntryCollector.java | 76 ++++++++++++++++ .../prometheus/impl/EntrySinkCollector.java | 45 ---------- .../impl/InstanceMetaCollector.java | 64 -------------- .../canal/prometheus/impl/MetaCollector.java | 77 +++++++++++++++++ ...PrometheusCanalEventDownStreamHandler.java | 50 ++--------- .../PrometheusClientInstanceProfiler.java | 47 +++++----- ...ometheusClientInstanceProfilerFactory.java | 15 ---- .../canal/prometheus/impl/SinkCollector.java | 73 ++++++++++++++++ ...toreCollector.java => StoreCollector.java} | 33 +++++-- .../netty/CanalServerWithNettyProfiler.java | 86 +++---------------- .../server/netty/ClientInstanceProfiler.java | 6 +- .../netty/ClientInstanceProfilerFactory.java | 10 --- .../listener/ChannelFutureAggregator.java | 2 +- 16 files changed, 346 insertions(+), 337 deletions(-) create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/InstanceRegistry.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java rename prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/{MemoryStoreCollector.java => StoreCollector.java} (66%) delete mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java index 388f8e59..6908a3bd 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -13,8 +13,6 @@ import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; -import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.profiler; - /** * @author Chuanyi Li */ @@ -24,30 +22,29 @@ public class CanalInstanceExports { public static final String DEST = "destination"; public static final String[] DEST_LABELS = {DEST}; public static final List DEST_LABEL_LIST = Collections.singletonList(DEST); - private final String destination; +// private final String destination; private Collector storeCollector; private Collector delayCollector; private Collector metaCollector; private Collector sinkCollector; private Collector parserCollector; - private CanalInstanceExports(CanalInstance instance) { - this.destination = instance.getDestination(); - initEventsMetrics(instance); - initStoreCollector(instance); - initMetaCollector(instance); - initSinkCollector(instance); - initParserCollector(instance); + public CanalInstanceExports() { +// this.destination = instance.getDestination(); +// initEventsMetrics(instance); +// initStoreCollector(instance); +// initMetaCollector(instance); +// initSinkCollector(instance); +// initParserCollector(instance); } - static CanalInstanceExports forInstance(CanalInstance instance) { - return new CanalInstanceExports(instance); - } +// static CanalInstanceExports forInstance(CanalInstance instance) { +// return new CanalInstanceExports(instance); +// } - void register() { - profiler().start(destination); + void register(CanalInstance instance) { if (delayCollector != null) { delayCollector.register(); } @@ -65,8 +62,7 @@ public class CanalInstanceExports { } } - void unregister() { - profiler().stop(destination); + void unregister(CanalInstance instance) { if (delayCollector != null) { CollectorRegistry.defaultRegistry.unregister(delayCollector); } @@ -89,28 +85,28 @@ public class CanalInstanceExports { if (sink instanceof EntryEventSink) { EntryEventSink entryEventSink = (EntryEventSink) sink; // TODO ensure not to add handler again - PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination); - entryEventSink.addHandler(handler); - delayCollector = handler.getCollector(); +// PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination); +// entryEventSink.addHandler(handler); +// delayCollector = handler.getCollector(); } else { logger.warn("This impl register metrics for only EntryEventSink, skip."); } } - private void initStoreCollector(CanalInstance instance) { - try { - storeCollector = new MemoryStoreCollector(instance.getEventStore(), destination); - } catch (CanalStoreException cse) { - logger.warn("Failed to register metrics for destination {}.", destination, cse); - } - } +// private void initStoreCollector(CanalInstance instance) { +// try { +// storeCollector = new StoreCollector(instance.getEventStore(), destination); +// } catch (CanalStoreException cse) { +// logger.warn("Failed to register metrics for destination {}.", destination, cse); +// } +// } private void initMetaCollector(CanalInstance instance) { - metaCollector = new InstanceMetaCollector(instance); + metaCollector = new MetaCollector(instance); } private void initSinkCollector(CanalInstance instance) { - sinkCollector = new EntrySinkCollector(instance.getEventSink(), instance.getDestination()); +// sinkCollector = new EntryCollector(instance.getEventSink(), instance.getDestination()); } private void initParserCollector(CanalInstance instance) { diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/InstanceRegistry.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/InstanceRegistry.java new file mode 100644 index 00000000..e320efc4 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/InstanceRegistry.java @@ -0,0 +1,14 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.instance.core.CanalInstance; + +/** + * @author Chuanyi Li + */ +public interface InstanceRegistry { + + void register(CanalInstance instance); + + void unregister(CanalInstance instance); + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java index 5d417883..c3a241bd 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -1,8 +1,8 @@ package com.alibaba.otter.canal.prometheus; import com.alibaba.otter.canal.instance.core.CanalInstance; -import com.alibaba.otter.canal.prometheus.impl.PrometheusClientInstanceProfilerFactory; -import com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler; +import com.alibaba.otter.canal.prometheus.impl.PrometheusClientInstanceProfiler; +import com.alibaba.otter.canal.server.netty.ClientInstanceProfiler; import com.alibaba.otter.canal.spi.CanalMetricsService; import io.prometheus.client.exporter.HTTPServer; import io.prometheus.client.hotspot.DefaultExports; @@ -10,10 +10,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.DISABLED; +import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.NOP; import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.profiler; /** @@ -21,10 +19,11 @@ import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler. */ public class PrometheusService implements CanalMetricsService { - private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); - private final Map exports = new ConcurrentHashMap(); - private volatile boolean running = false; + private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); + private final CanalInstanceExports instanceExports = new CanalInstanceExports(); + private volatile boolean running = false; private HTTPServer server; + private ClientInstanceProfiler clientProfiler = new PrometheusClientInstanceProfiler(); private PrometheusService() { } @@ -52,7 +51,10 @@ public class PrometheusService implements CanalMetricsService { DefaultExports.initialize(); // Canal server level exports CanalServerExports.initialize(); - profiler().setInstanceProfilerFactory(new PrometheusClientInstanceProfilerFactory()); + if (!clientProfiler.isStart()) { + clientProfiler.start(); + } + profiler().setInstanceProfiler(clientProfiler); } catch (Throwable t) { logger.warn("Unable to initialize server exports.", t); } @@ -63,13 +65,11 @@ public class PrometheusService implements CanalMetricsService { @Override public void terminate() { running = false; - // Normally, service should be terminated at canal shutdown. - // No need to unregister instance exports explicitly. - // But for the sake of safety, unregister them. - for (CanalInstanceExports cie : exports.values()) { - cie.unregister(); + instanceExports.unregister(); + if (clientProfiler.isStart()) { + clientProfiler.stop(); } - profiler().setInstanceProfilerFactory(DISABLED); + profiler().setInstanceProfiler(NOP); if (server != null) { server.stop(); } @@ -111,4 +111,5 @@ public class PrometheusService implements CanalMetricsService { } logger.info("Unregister metrics for destination {}.", instance.getDestination()); } + } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java new file mode 100644 index 00000000..83e4ef95 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java @@ -0,0 +1,76 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.prometheus.InstanceRegistry; +import com.alibaba.otter.canal.sink.CanalEventSink; +import com.alibaba.otter.canal.sink.entry.EntryEventSink; +import com.google.common.base.Preconditions; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; + +/** + * @author Chuanyi Li + */ +public class EntryCollector extends Collector implements InstanceRegistry { + + private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); + private static final String DELAY_NAME = "canal_instance_traffic_delay"; + private static final String delayHelpName = "Traffic delay of canal instance"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + + @Override + public List collect() { + List mfs = new ArrayList(); + GaugeMetricFamily delay = new GaugeMetricFamily(DELAY_NAME, + delayHelpName, DEST_LABEL_LIST); + for (EntryMetricsHolder emh : instances.values()) { + delay.addMetric(emh.destLabelValues, emh.latestExecTime.doubleValue()); + } + mfs.add(delay); + return mfs; + } + + @Override + public void register(CanalInstance instance) { + final String destination = instance.getDestination(); + EntryMetricsHolder holder = new EntryMetricsHolder(); + holder.destLabelValues = Collections.singletonList(destination); + CanalEventSink sink = instance.getEventSink(); + if (!(sink instanceof EntryEventSink)) { + throw new IllegalArgumentException("CanalEventSink must be EntryEventSink"); + } + EntryEventSink entrySink = (EntryEventSink) sink; + PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(); + holder.latestExecTime = handler.getLatestExecuteTime(); + entrySink.addHandler(handler, 0); + Preconditions.checkNotNull(holder.destLabelValues); + Preconditions.checkNotNull(holder.latestExecTime); + EntryMetricsHolder old = instances.putIfAbsent(destination, holder); + if (old != null) { + logger.warn("Ignore repeated EntryCollector register for instance {}.", destination); + } + } + + @Override + public void unregister(CanalInstance instance) { + final String destination = instance.getDestination(); + instances.remove(destination); + } + + private class EntryMetricsHolder { + private AtomicLong latestExecTime; + private List destLabelValues; + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java deleted file mode 100644 index 54775842..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntrySinkCollector.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.alibaba.otter.canal.prometheus.impl; - -import com.alibaba.otter.canal.sink.CanalEventSink; -import com.alibaba.otter.canal.sink.entry.EntryEventSink; -import io.prometheus.client.Collector; -import io.prometheus.client.CounterMetricFamily; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; - -/** - * @author Chuanyi Li - */ -public class EntrySinkCollector extends Collector { - - private static final String SINK_BLOCKING_TIME = "canal_instance_sink_blocking_time"; - private final String sinkBlockTimeHelp; - private final List destLabelValues; - private final AtomicLong eventsSinkBlockingTime; - - - public EntrySinkCollector(CanalEventSink sink, String destination) { - if (!(sink instanceof EntryEventSink)) { - throw new IllegalArgumentException("CanalEventSink must be EntryEventSink"); - } - this.destLabelValues = Collections.singletonList(destination); - EntryEventSink entrySink = (EntryEventSink) sink; - this.sinkBlockTimeHelp = "Sink blocking time of instance " + destination; - eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime(); - } - - @Override public List collect() { - List mfs = new ArrayList(); - CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME, - sinkBlockTimeHelp, DEST_LABEL_LIST); - blockingCounter.addMetric(destLabelValues, eventsSinkBlockingTime.doubleValue()); - mfs.add(blockingCounter); - return mfs; - } - -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java deleted file mode 100644 index 2e8b7fc4..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.alibaba.otter.canal.prometheus.impl; - -import com.alibaba.otter.canal.instance.core.CanalInstance; -import com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring; -import com.alibaba.otter.canal.meta.CanalMetaManager; -import com.alibaba.otter.canal.prometheus.CanalInstanceExports; -import com.alibaba.otter.canal.protocol.ClientIdentity; -import io.prometheus.client.Collector; -import io.prometheus.client.GaugeMetricFamily; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author Chuanyi Li - */ -public class InstanceMetaCollector extends Collector { - - private static final List InfoLabel = Arrays.asList("destination", "mode"); - - private CanalMetaManager metaManager; - - private final String destination; - - private final String mode; - - private final String subsHelp; - - public InstanceMetaCollector(CanalInstance instance) { - if (instance == null) { - throw new IllegalArgumentException("CanalInstance must not be null."); - } - if (instance instanceof CanalInstanceWithSpring) { - mode = "spring"; - } else { - mode = "manager"; - } - this.metaManager = instance.getMetaManager(); - this.destination = instance.getDestination(); - this.subsHelp = "Subscriptions of canal instance " + destination; - } - - @Override - public List collect() { - List mfs = new ArrayList(); - GaugeMetricFamily instanceInfo = new GaugeMetricFamily( - "canal_instance", - "Canal instance", - InfoLabel); - instanceInfo.addMetric(Arrays.asList(destination, mode), 1); - mfs.add(instanceInfo); - if (metaManager.isStart()) { - // client id = hardcode 1001, 目前没有意义 - List subs = metaManager.listAllSubscribeInfo(destination); - GaugeMetricFamily subscriptions = new GaugeMetricFamily( - "canal_instance_subscription", - subsHelp, CanalInstanceExports.DEST_LABEL_LIST); - subscriptions.addMetric(Arrays.asList(destination), subs.size()); - mfs.add(subscriptions); - } - return mfs; - } -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java new file mode 100644 index 00000000..eb5a2bde --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java @@ -0,0 +1,77 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring; +import com.alibaba.otter.canal.meta.CanalMetaManager; +import com.alibaba.otter.canal.prometheus.InstanceRegistry; +import com.alibaba.otter.canal.protocol.ClientIdentity; +import com.google.common.base.Preconditions; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Chuanyi Li + */ +public class MetaCollector extends Collector implements InstanceRegistry { + + private static final List InfoLabel = Arrays.asList("destination", "mode"); + private static final Logger logger = LoggerFactory.getLogger(MetaCollector.class); + private static final String SUBSCRIPTION = "canal_instance"; + private static final String subscriptionHelp = "Canal instance"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + + @Override + public List collect() { + List mfs = new ArrayList(); + GaugeMetricFamily instanceInfo = new GaugeMetricFamily( + SUBSCRIPTION, + subscriptionHelp, + InfoLabel); + for (Map.Entry nme : instances.entrySet()) { + final String destination = nme.getKey(); + final MetaMetricsHolder nmh = nme.getValue(); + List subs = nmh.metaManager.listAllSubscribeInfo(destination); + int count = subs == null ? 0 : subs.size(); + instanceInfo.addMetric(nmh.infoLabelValues, count); + } + mfs.add(instanceInfo); + return mfs; + } + + @Override + public void register(CanalInstance instance) { + final String destination = instance.getDestination(); + MetaMetricsHolder holder = new MetaMetricsHolder(); + String mode = (instance instanceof CanalInstanceWithSpring) ? "spring" : "manager"; + holder.infoLabelValues = Arrays.asList(destination, mode); + holder.metaManager = instance.getMetaManager(); + Preconditions.checkNotNull(holder.infoLabelValues); + Preconditions.checkNotNull(holder.metaManager); + MetaMetricsHolder old = instances.putIfAbsent(destination, holder); + if (old != null) { + logger.warn("Ignore repeated MetaCollector register for instance {}.", destination); + } + } + + @Override + public void unregister(CanalInstance instance) { + final String destination = instance.getDestination(); + instances.remove(destination); + } + + private class MetaMetricsHolder { + private List infoLabelValues; + private CanalMetaManager metaManager; + } + + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java index 47d4824d..763ce46b 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java @@ -1,53 +1,16 @@ package com.alibaba.otter.canal.prometheus.impl; -import com.alibaba.otter.canal.prometheus.CanalInstanceExports; import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler; import com.alibaba.otter.canal.store.model.Event; -import io.prometheus.client.Collector; -import io.prometheus.client.GaugeMetricFamily; - -import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; /** * @author Chuanyi Li */ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler> { - private final Collector collector; - - private long latestExecuteTime = 0L; - - private static final String DELAY_NAME = "canal_instance_traffic_delay"; - - private final String delayHelpName; - - private final List labelValues; - - public PrometheusCanalEventDownStreamHandler(final String destination) { - this.delayHelpName = "Traffic delay of canal instance " + destination + " in seconds."; - this.labelValues = Collections.singletonList(destination); - collector = new Collector() { - @Override - public List collect() { - List mfs = new ArrayList(); - long now = System.currentTimeMillis(); - GaugeMetricFamily delay = new GaugeMetricFamily( - DELAY_NAME, - delayHelpName, - CanalInstanceExports.DEST_LABEL_LIST); - double d = 0.0; - if (latestExecuteTime > 0) { - d = now - latestExecuteTime; - } - d = d > 0.0 ? (d / 1000) : 0.0; - delay.addMetric(labelValues, d); - mfs.add(delay); - return mfs; - } - }; - } + private final AtomicLong latestExecuteTime = new AtomicLong(0L); @Override public List before(List events) { @@ -59,15 +22,16 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow if (events != null && !events.isEmpty()) { Event last = events.get(events.size() - 1); long ts = last.getExecuteTime(); - if (ts > latestExecuteTime) { - latestExecuteTime = ts; + long ls = latestExecuteTime.get(); + if (ts > ls) { + latestExecuteTime.lazySet(ts); } } return events; } - public Collector getCollector() { - return this.collector; + public AtomicLong getLatestExecuteTime() { + return latestExecuteTime; } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java index 4cbbdb89..331897ee 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java @@ -20,68 +20,60 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler private static final String EMPTY_BATCHES = "canal_instance_client_empty_batches"; private static final String ERRORS = "canal_instance_client_request_error"; private static final String LATENCY = "canal_instance_client_request_latency"; + private final Counter outboundCounter; + private final Counter packetsCounter; + private final Counter emptyBatchesCounter; + private final Counter errorsCounter; + private final Histogram responseLatency; private volatile boolean running = false; - private final String destination; - private final String[] destLabelValues; - private final Counter outboundCounter; - private final Counter packetsCounter; - private final Counter emptyBatchesCounter; - private final Counter errorsCounter; - private final Histogram responseLatency; - public PrometheusClientInstanceProfiler(String destination) { - this.destination = destination; - this.destLabelValues = new String[]{destination}; + public PrometheusClientInstanceProfiler() { this.outboundCounter = Counter.build() .labelNames(DEST_LABELS) .name(OUTBOUND_BYTES) - .help("Send bytes to client of instance " + destination) + .help("Total bytes sent to client.") .create(); this.packetsCounter = Counter.build() .labelNames(new String[]{DEST, "packetType"}) .name(PACKET_TYPE) - .help("Send packets to client of instance " + destination) + .help("Total packets sent to client.") .create(); this.emptyBatchesCounter = Counter.build() .labelNames(DEST_LABELS) .name(EMPTY_BATCHES) - .help("Send empty batches to client of instance " + destination) + .help("Total empty batches sent to client.") .create(); this.errorsCounter = Counter.build() .labelNames(new String[]{DEST, "errorCode"}) .name(ERRORS) - .help("Client request errors of instance " + destination) + .help("Total client request errors.") .create(); this.responseLatency = Histogram.build() .labelNames(DEST_LABELS) .name(LATENCY) - .help("Client request latency of instance " + destination) + .help("Client request latency.") // buckets in milliseconds .buckets(1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0) .create(); } - @Override - public String getDestination() { - return this.destination; - } - @Override public void profiling(ClientRequestResult result) { + String destination = result.getDestination(); PacketType type = result.getType(); - outboundCounter.labels(destLabelValues).inc(result.getAmount()); + outboundCounter.labels(destination).inc(result.getAmount()); packetsCounter.labels(destination, type.name()).inc(); short errorCode = result.getErrorCode(); if (errorCode > 0) { errorsCounter.labels(destination, Short.toString(errorCode)).inc(); } long latency = result.getLatency(); - responseLatency.labels(destLabelValues).observe(latency / 1000000); + responseLatency.labels(destination).observe(latency / 1000000); switch (type) { case GET: boolean empty = result.getEmpty(); if (empty) { - emptyBatchesCounter.labels(destLabelValues).inc(); + emptyBatchesCounter.labels(destination).inc(); } break; // reserve for others @@ -90,7 +82,8 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler } } - @Override public void start() { + @Override + public void start() { if (outboundCounter != null) { outboundCounter.register(); } @@ -109,7 +102,8 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler running = true; } - @Override public void stop() { + @Override + public void stop() { running = false; if (outboundCounter != null) { CollectorRegistry.defaultRegistry.unregister(outboundCounter); @@ -128,7 +122,8 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler } } - @Override public boolean isStart() { + @Override + public boolean isStart() { return running; } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java deleted file mode 100644 index 6f80c221..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfilerFactory.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.alibaba.otter.canal.prometheus.impl; - -import com.alibaba.otter.canal.server.netty.ClientInstanceProfiler; -import com.alibaba.otter.canal.server.netty.ClientInstanceProfilerFactory; - -/** - * @author Chuanyi Li - */ -public class PrometheusClientInstanceProfilerFactory implements ClientInstanceProfilerFactory { - - @Override - public ClientInstanceProfiler create(String destination) { - return new PrometheusClientInstanceProfiler(destination); - } -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java new file mode 100644 index 00000000..11903314 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java @@ -0,0 +1,73 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.prometheus.InstanceRegistry; +import com.alibaba.otter.canal.sink.CanalEventSink; +import com.alibaba.otter.canal.sink.entry.EntryEventSink; +import com.google.common.base.Preconditions; +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; + +/** + * @author Chuanyi Li + */ +public class SinkCollector extends Collector implements InstanceRegistry { + + private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); + private static final String SINK_BLOCKING_TIME = "canal_instance_sink_blocking_time"; + private static final String sinkBlockTimeHelp = "Total sink blocking time"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + + @Override + public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME, + sinkBlockTimeHelp, DEST_LABEL_LIST); + for (SinkMetricsHolder smh : instances.values()) { + blockingCounter.addMetric(smh.destLabelValues, smh.eventsSinkBlockingTime.doubleValue()); + } + mfs.add(blockingCounter); + return mfs; + } + + @Override + public void register(CanalInstance instance) { + final String destination = instance.getDestination(); + SinkMetricsHolder holder = new SinkMetricsHolder(); + holder.destLabelValues = Collections.singletonList(destination); + CanalEventSink sink = instance.getEventSink(); + if (!(sink instanceof EntryEventSink)) { + throw new IllegalArgumentException("CanalEventSink must be EntryEventSink"); + } + EntryEventSink entrySink = (EntryEventSink) sink; + holder.eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime(); + Preconditions.checkNotNull(holder.destLabelValues); + Preconditions.checkNotNull(holder.eventsSinkBlockingTime); + SinkMetricsHolder old = instances.putIfAbsent(destination, holder); + if (old != null) { + logger.warn("Ignore repeated SinkCollector register for instance {}.", destination); + } + } + + @Override + public void unregister(CanalInstance instance) { + final String destination = instance.getDestination(); + instances.remove(destination); + } + + private class SinkMetricsHolder { + private AtomicLong eventsSinkBlockingTime; + private List destLabelValues; + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java similarity index 66% rename from prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java rename to prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java index 4c212c4c..3784af9b 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java @@ -1,6 +1,8 @@ package com.alibaba.otter.canal.prometheus.impl; +import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.prometheus.InstanceRegistry; import com.alibaba.otter.canal.store.CanalEventStore; import com.alibaba.otter.canal.store.CanalStoreException; import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; @@ -12,21 +14,24 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; /** * @author Chuanyi Li */ -public class MemoryStoreCollector extends Collector { +public class StoreCollector extends Collector implements InstanceRegistry { private static final Class clazz = MemoryEventStoreWithBuffer.class; - private final String destination; - private final AtomicLong putSequence; - private final AtomicLong ackSequence; - private final String putHelp; - private final String ackHelp; + private static final String PRODUCE = "canal_instance_store_produce_seq"; + private static final String CONSUME = "canal_instance_store_consume_seq"; + private static final String produceHelp = "Produced sequence of canal instance"; + private static final String consumeHelp = "Consumed sequence of canal instance"; + private final ConcurrentMap instances = new ConcurrentHashMap(); - public MemoryStoreCollector(CanalEventStore store, String destination) { + + public StoreCollector(CanalEventStore store, String destination) { this.destination = destination; if (!(store instanceof MemoryEventStoreWithBuffer)) { throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer"); @@ -34,8 +39,6 @@ public class MemoryStoreCollector extends Collector { MemoryEventStoreWithBuffer ms = (MemoryEventStoreWithBuffer) store; putSequence = getDeclaredValue(ms, "putSequence"); ackSequence = getDeclaredValue(ms, "ackSequence"); - putHelp = "Produced sequence of canal instance " + destination; - ackHelp = "Consumed sequence of canal instance " + destination; } @Override @@ -67,4 +70,16 @@ public class MemoryStoreCollector extends Collector { return value; } + @Override public void register(CanalInstance instance) { + + } + + @Override public void unregister(CanalInstance instance) { + + } + + private class StoreMetricsHolder { + private AtomicLong putSeq; + private AtomicLong ackSeq; + } } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java index e44476d7..cca3fa6b 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java @@ -1,105 +1,39 @@ package com.alibaba.otter.canal.server.netty; -import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator.ClientRequestResult; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; /** * @author Chuanyi Li */ public class CanalServerWithNettyProfiler { - public static final ClientInstanceProfilerFactory DISABLED = new DefaultClientInstanceProfilerFactory(); - private volatile ClientInstanceProfilerFactory clientInstanceProfilerFactory; - private final ConcurrentMap cliPfs; - private final CanalServerWithEmbedded server; + public static final ClientInstanceProfiler NOP = new DefaultClientInstanceProfiler(); + private ClientInstanceProfiler instanceProfiler; private static class SingletonHolder { private static CanalServerWithNettyProfiler SINGLETON = new CanalServerWithNettyProfiler(); } private CanalServerWithNettyProfiler() { - this.clientInstanceProfilerFactory = DISABLED; - this.cliPfs = new ConcurrentHashMap(); - this.server = CanalServerWithEmbedded.instance(); + this.instanceProfiler = NOP; } public static CanalServerWithNettyProfiler profiler() { return SingletonHolder.SINGLETON; } - public void profiling(String destination, ClientRequestResult result) { - if (isDisabled()) { - return; - } - ClientInstanceProfiler profiler = cliPfs.get(destination); - if (profiler != null) { - profiler.profiling(result); - } + public void profiling(ClientRequestResult result) { + instanceProfiler.profiling(result); } - public void start(String destination) { - if (isDisabled()) { - return; - } - if (server.isStart(destination)) { - throw new IllegalStateException("Instance profiler should not be start while running."); - } - ClientInstanceProfiler profiler = clientInstanceProfilerFactory.create(destination); - profiler.start(); - cliPfs.put(destination, profiler); + public void setInstanceProfiler(ClientInstanceProfiler instanceProfiler) { + this.instanceProfiler = instanceProfiler; } - /** - * Remove instance profiler for specified instance. - * Only accepted while instance is not running. - * @param destination canal instance destination - */ - public void stop(String destination) { - if (isDisabled()) { - return; - } - if (server.isStart(destination)) { - throw new IllegalStateException("Instance profiler should not be stop while running."); - } - ClientInstanceProfiler profiler = cliPfs.remove(destination); - if (profiler != null && profiler.isStart()) { - profiler.stop(); - } - } - - public void setInstanceProfilerFactory(ClientInstanceProfilerFactory factory) { - this.clientInstanceProfilerFactory = factory; - } - - private boolean isDisabled() { - return clientInstanceProfilerFactory == DISABLED || clientInstanceProfilerFactory == null; - } - - private ClientInstanceProfiler tryGet(String destination) { - //try fast get - ClientInstanceProfiler profiler = cliPfs.get(destination); - if (profiler == null) { - synchronized (cliPfs) { - if (server.isStart(destination)) { - // avoid overwriting - cliPfs.putIfAbsent(destination, clientInstanceProfilerFactory.create(destination)); - profiler = cliPfs.get(destination); - if (!profiler.isStart()) { - profiler.start(); - } - } - } - } - return profiler; - } - - private static class DefaultClientInstanceProfilerFactory implements ClientInstanceProfilerFactory { + private static class DefaultClientInstanceProfiler extends AbstractCanalLifeCycle implements ClientInstanceProfiler { @Override - public ClientInstanceProfiler create(String destination) { - throw new UnsupportedOperationException(); - } + public void profiling(ClientRequestResult result) {} } } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java index bd4e2dbc..2ffbd46e 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java @@ -1,15 +1,13 @@ package com.alibaba.otter.canal.server.netty; import com.alibaba.otter.canal.common.CanalLifeCycle; -import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator; +import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator.ClientRequestResult; /** * @author Chuanyi Li */ public interface ClientInstanceProfiler extends CanalLifeCycle { - String getDestination(); - - void profiling(ChannelFutureAggregator.ClientRequestResult result); + void profiling(ClientRequestResult result); } diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java deleted file mode 100644 index 12b54bc6..00000000 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfilerFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.alibaba.otter.canal.server.netty; - -/** - * @author Chuanyi Li - */ -public interface ClientInstanceProfilerFactory { - - ClientInstanceProfiler create(String destination); - -} diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java index 89755431..b962b9a0 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java @@ -45,7 +45,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener { if (future.getCause() != null) { result.channelError = future.getCause(); } - profiler().profiling(result.destination, result); + profiler().profiling(result); } /** From 40c3e65341871a315acc7c1374a7dec0f77066bb Mon Sep 17 00:00:00 2001 From: Chuanyi Li L Date: Tue, 7 Aug 2018 17:17:33 +0800 Subject: [PATCH 20/21] Refactor metrics code. --- .../common/utils/SerializedLongAdder.java | 31 ---- .../parse/inbound/AbstractEventParser.java | 1 - .../mysql/AbstractMysqlEventParser.java | 6 +- .../parse/inbound/mysql/MysqlConnection.java | 8 +- .../parse/inbound/mysql/MysqlEventParser.java | 7 +- .../mysql/MysqlMultiStageCoprocessor.java | 8 +- .../prometheus/CanalInstanceExports.java | 134 +++++++----------- .../canal/prometheus/CanalServerExports.java | 17 --- .../canal/prometheus/PrometheusService.java | 33 +++-- .../canal/prometheus/impl/EntryCollector.java | 78 ++++++++-- .../canal/prometheus/impl/MetaCollector.java | 27 ++-- .../prometheus/impl/MysqlParserCollector.java | 74 ---------- .../prometheus/impl/ParserCollector.java | 114 +++++++++++++++ ...PrometheusCanalEventDownStreamHandler.java | 67 +++++++-- .../canal/prometheus/impl/SinkCollector.java | 27 ++-- .../canal/prometheus/impl/StoreCollector.java | 131 +++++++++++------ .../memory/MemoryEventStoreWithBuffer.java | 19 +++ 17 files changed, 462 insertions(+), 320 deletions(-) delete mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java delete mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java create mode 100644 prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java deleted file mode 100644 index 4d86a99e..00000000 --- a/common/src/main/java/com/alibaba/otter/canal/common/utils/SerializedLongAdder.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.alibaba.otter.canal.common.utils; - -/** - * 用于dump记录counter - * 这种场景下， - * 写操作>>读操作 - * dump线程为串行，读取counter时，并不严格要求最新值可见，使用普通long即可。 - * - * @author Chuanyi Li - */ -public class SerializedLongAdder { - - private long value = 0; - - public SerializedLongAdder(long initial) { - this.value = initial; - } - - public void add(long x) { - value += x; - } - - public void increment() { - value ++; - } - - public long get() { - return value; - } - -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index ff92dd9d..a83c3516 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -8,7 +8,6 @@ import java.util.TimerTask; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang.math.RandomUtils; diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 5e3c6b02..156d4692 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -1,8 +1,8 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicLong; -import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +35,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { protected boolean filterRows = false; protected boolean filterTableError = false; protected boolean useDruidDdlFilter = true; - private final SerializedLongAdder eventsPublishBlockingTime = new SerializedLongAdder(0L); + private final AtomicLong eventsPublishBlockingTime = new AtomicLong(0L); protected BinlogParser buildParser() { LogEventConvert convert = new LogEventConvert(); @@ -192,7 +192,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { } } - public SerializedLongAdder getEventsPublishBlockingTime() { + public AtomicLong getEventsPublishBlockingTime() { return this.eventsPublishBlockingTime; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index c125dbd3..9d89a145 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -7,8 +7,8 @@ import java.net.InetSocketAddress; import java.nio.charset.Charset; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; -import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +50,7 @@ public class MysqlConnection implements ErosaConnection { private AuthenticationInfo authInfo; protected int connTimeout = 5 * 1000; // 5秒 protected int soTimeout = 60 * 60 * 1000; // 1小时 - private SerializedLongAdder receivedBinlogBytes; + private AtomicLong receivedBinlogBytes; public MysqlConnection(){ } @@ -461,7 +461,7 @@ public class MysqlConnection implements ErosaConnection { private void accumulateReceivedBytes(long x) { if (receivedBinlogBytes != null) { - receivedBinlogBytes.add(x); + receivedBinlogBytes.addAndGet(x); } } @@ -606,7 +606,7 @@ public class MysqlConnection implements ErosaConnection { this.authInfo = authInfo; } - public void setReceivedBinlogBytes(SerializedLongAdder receivedBinlogBytes) { + public void setReceivedBinlogBytes(AtomicLong receivedBinlogBytes) { this.receivedBinlogBytes = receivedBinlogBytes; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index a87be542..69cf0572 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -10,7 +10,6 @@ import java.util.Map; import java.util.TimerTask; import java.util.concurrent.atomic.AtomicLong; -import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import org.springframework.util.CollectionUtils; @@ -74,7 +73,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 // instance received binlog bytes - private final SerializedLongAdder receivedBinlogBytes = new SerializedLongAdder(0L); + private final AtomicLong receivedBinlogBytes = new AtomicLong(0L); protected ErosaConnection buildErosaConnection() { return buildMysqlConnection(this.runningInfo); @@ -517,7 +516,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException { // 针对开始的第一条为非Begin记录，需要从该binlog扫描 - final AtomicLong preTransactionStartPosition = new AtomicLong(0L); + final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L); mysqlConnection.reconnect(); mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction() { @@ -918,7 +917,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE - public SerializedLongAdder getReceivedBinlogBytes() { + public AtomicLong getReceivedBinlogBytes() { return this.receivedBinlogBytes; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index e3206e3e..92447d50 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -2,9 +2,9 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; -import com.alibaba.otter.canal.common.utils.SerializedLongAdder; import org.apache.commons.lang.StringUtils; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; @@ -65,7 +65,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement private ExecutorService stageExecutor; private String destination; private volatile CanalParseException exception; - private SerializedLongAdder eventsPublishBlockingTime; + private AtomicLong eventsPublishBlockingTime; public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert, EventTransactionBuffer transactionBuffer, String destination){ @@ -176,7 +176,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement } disruptorMsgBuffer.publish(next); if (fullTimes > 0) { - eventsPublishBlockingTime.add(System.nanoTime() - blockingStart); + eventsPublishBlockingTime.addAndGet(System.nanoTime() - blockingStart); } break; } catch (InsufficientCapacityException e) { @@ -451,7 +451,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement this.connection = connection; } - public void setEventsPublishBlockingTime(SerializedLongAdder eventsPublishBlockingTime) { + public void setEventsPublishBlockingTime(AtomicLong eventsPublishBlockingTime) { this.eventsPublishBlockingTime = eventsPublishBlockingTime; } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java index 6908a3bd..487855fa 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -2,9 +2,6 @@ package com.alibaba.otter.canal.prometheus; import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.prometheus.impl.*; -import com.alibaba.otter.canal.sink.CanalEventSink; -import com.alibaba.otter.canal.sink.entry.EntryEventSink; -import com.alibaba.otter.canal.store.CanalStoreException; import io.prometheus.client.Collector; import io.prometheus.client.CollectorRegistry; import org.slf4j.Logger; @@ -18,98 +15,71 @@ import java.util.List; */ public class CanalInstanceExports { - private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); - public static final String DEST = "destination"; - public static final String[] DEST_LABELS = {DEST}; - public static final List DEST_LABEL_LIST = Collections.singletonList(DEST); -// private final String destination; - private Collector storeCollector; - private Collector delayCollector; - private Collector metaCollector; - private Collector sinkCollector; - private Collector parserCollector; + private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); + public static final String DEST = "destination"; + public static final String[] DEST_LABELS = {DEST}; + public static final List DEST_LABELS_LIST = Collections.singletonList(DEST); + private final Collector storeCollector; + private final Collector entryCollector; + private final Collector metaCollector; + private final Collector sinkCollector; + private final Collector parserCollector; - public CanalInstanceExports() { -// this.destination = instance.getDestination(); -// initEventsMetrics(instance); -// initStoreCollector(instance); -// initMetaCollector(instance); -// initSinkCollector(instance); -// initParserCollector(instance); + private CanalInstanceExports() { + this.storeCollector = StoreCollector.instance(); + this.entryCollector = EntryCollector.instance(); + this.metaCollector = MetaCollector.instance(); + this.sinkCollector = SinkCollector.instance(); + this.parserCollector = ParserCollector.instance(); } + private static class SingletonHolder { + private static final CanalInstanceExports SINGLETON = new CanalInstanceExports(); + } + public static CanalInstanceExports instance() { + return SingletonHolder.SINGLETON; + } -// static CanalInstanceExports forInstance(CanalInstance instance) { -// return new CanalInstanceExports(instance); -// } + public void initialize() { + storeCollector.register(); + entryCollector.register(); + metaCollector.register(); + sinkCollector.register(); + parserCollector.register(); + } + + public void terminate() { + CollectorRegistry.defaultRegistry.unregister(storeCollector); + CollectorRegistry.defaultRegistry.unregister(entryCollector); + CollectorRegistry.defaultRegistry.unregister(metaCollector); + CollectorRegistry.defaultRegistry.unregister(sinkCollector); + CollectorRegistry.defaultRegistry.unregister(parserCollector); + } void register(CanalInstance instance) { - if (delayCollector != null) { - delayCollector.register(); - } - if (storeCollector != null) { - storeCollector.register(); - } - if (metaCollector != null) { - metaCollector.register(); - } - if (sinkCollector != null) { - sinkCollector.register(); - } - if (parserCollector != null) { - parserCollector.register(); - } + requiredInstanceRegistry(storeCollector).register(instance); + requiredInstanceRegistry(entryCollector).register(instance); + requiredInstanceRegistry(metaCollector).register(instance); + requiredInstanceRegistry(sinkCollector).register(instance); + requiredInstanceRegistry(parserCollector).register(instance); + logger.info("Successfully register metrics for instance {}.", instance.getDestination()); } void unregister(CanalInstance instance) { - if (delayCollector != null) { - CollectorRegistry.defaultRegistry.unregister(delayCollector); - } - if (storeCollector != null) { - CollectorRegistry.defaultRegistry.unregister(storeCollector); - } - if (metaCollector != null) { - CollectorRegistry.defaultRegistry.unregister(metaCollector); - } - if (sinkCollector != null) { - CollectorRegistry.defaultRegistry.unregister(sinkCollector); - } - if (parserCollector != null) { - CollectorRegistry.defaultRegistry.unregister(parserCollector); - } + requiredInstanceRegistry(storeCollector).unregister(instance); + requiredInstanceRegistry(entryCollector).unregister(instance); + requiredInstanceRegistry(metaCollector).unregister(instance); + requiredInstanceRegistry(sinkCollector).unregister(instance); + requiredInstanceRegistry(parserCollector).unregister(instance); + logger.info("Successfully unregister metrics for instance {}.", instance.getDestination()); } - private void initEventsMetrics(CanalInstance instance) { - CanalEventSink sink = instance.getEventSink(); - if (sink instanceof EntryEventSink) { - EntryEventSink entryEventSink = (EntryEventSink) sink; - // TODO ensure not to add handler again -// PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination); -// entryEventSink.addHandler(handler); -// delayCollector = handler.getCollector(); - } else { - logger.warn("This impl register metrics for only EntryEventSink, skip."); + private InstanceRegistry requiredInstanceRegistry(Collector collector) { + if (!(collector instanceof InstanceRegistry)) { + throw new IllegalArgumentException("Canal collector need to implement InstanceRegistry."); } + return (InstanceRegistry) collector; } -// private void initStoreCollector(CanalInstance instance) { -// try { -// storeCollector = new StoreCollector(instance.getEventStore(), destination); -// } catch (CanalStoreException cse) { -// logger.warn("Failed to register metrics for destination {}.", destination, cse); -// } -// } - - private void initMetaCollector(CanalInstance instance) { - metaCollector = new MetaCollector(instance); - } - - private void initSinkCollector(CanalInstance instance) { -// sinkCollector = new EntryCollector(instance.getEventSink(), instance.getDestination()); - } - - private void initParserCollector(CanalInstance instance) { - parserCollector = new MysqlParserCollector(instance.getEventParser(), instance.getDestination()); - } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java deleted file mode 100644 index ded44fb8..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.alibaba.otter.canal.prometheus; - - -/** - * @author Chuanyi Li - */ -public class CanalServerExports { - - private static boolean initialized = false; - - public static synchronized void initialize() { - if (!initialized) { - initialized = true; - } - } - -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java index c3a241bd..9e3ef18c 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -20,12 +20,13 @@ import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler. public class PrometheusService implements CanalMetricsService { private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); - private final CanalInstanceExports instanceExports = new CanalInstanceExports(); + private final CanalInstanceExports instanceExports; private volatile boolean running = false; private HTTPServer server; private ClientInstanceProfiler clientProfiler = new PrometheusClientInstanceProfiler(); private PrometheusService() { + this.instanceExports = CanalInstanceExports.instance(); } private static class SingletonHolder { @@ -49,8 +50,7 @@ public class PrometheusService implements CanalMetricsService { try { // JVM exports DefaultExports.initialize(); - // Canal server level exports - CanalServerExports.initialize(); + instanceExports.initialize(); if (!clientProfiler.isStart()) { clientProfiler.start(); } @@ -65,13 +65,17 @@ public class PrometheusService implements CanalMetricsService { @Override public void terminate() { running = false; - instanceExports.unregister(); - if (clientProfiler.isStart()) { - clientProfiler.stop(); - } - profiler().setInstanceProfiler(NOP); - if (server != null) { - server.stop(); + try { + instanceExports.terminate(); + if (clientProfiler.isStart()) { + clientProfiler.stop(); + } + profiler().setInstanceProfiler(NOP); + if (server != null) { + server.stop(); + } + } catch (Throwable t) { + logger.warn("Something happened while terminating.", t); } } @@ -87,9 +91,7 @@ public class PrometheusService implements CanalMetricsService { return; } try { - CanalInstanceExports export = CanalInstanceExports.forInstance(instance); - export.register(); - exports.put(instance.getDestination(), export); + instanceExports.register(instance); } catch (Throwable t) { logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t); } @@ -102,10 +104,7 @@ public class PrometheusService implements CanalMetricsService { logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination()); } try { - CanalInstanceExports export = exports.remove(instance.getDestination()); - if (export != null) { - export.unregister(); - } + instanceExports.unregister(instance); } catch (Throwable t) { logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t); } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java index 83e4ef95..dd282065 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java @@ -2,10 +2,12 @@ package com.alibaba.otter.canal.prometheus.impl; import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.prometheus.InstanceRegistry; +import com.alibaba.otter.canal.sink.CanalEventDownStreamHandler; import com.alibaba.otter.canal.sink.CanalEventSink; import com.alibaba.otter.canal.sink.entry.EntryEventSink; import com.google.common.base.Preconditions; import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; import io.prometheus.client.GaugeMetricFamily; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,27 +19,43 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; /** * @author Chuanyi Li */ public class EntryCollector extends Collector implements InstanceRegistry { - private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); - private static final String DELAY_NAME = "canal_instance_traffic_delay"; - private static final String delayHelpName = "Traffic delay of canal instance"; - private final ConcurrentMap instances = new ConcurrentHashMap(); + private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); + private static final String DELAY = "canal_instance_traffic_delay"; + private static final String TRANSACTION = "canal_instance_transactions"; + private static final String DELAY_HELP = "Traffic delay of canal instance"; + private static final String TRANSACTION_HELP = "Transactions counter of canal instance"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + + private EntryCollector() {} + + private static class SingletonHolder { + private static final EntryCollector SINGLETON = new EntryCollector(); + } + + public static EntryCollector instance() { + return SingletonHolder.SINGLETON; + } @Override public List collect() { List mfs = new ArrayList(); - GaugeMetricFamily delay = new GaugeMetricFamily(DELAY_NAME, - delayHelpName, DEST_LABEL_LIST); + GaugeMetricFamily delay = new GaugeMetricFamily(DELAY, + DELAY_HELP, DEST_LABELS_LIST); + CounterMetricFamily transactions = new CounterMetricFamily(TRANSACTION, + TRANSACTION_HELP, DEST_LABELS_LIST); for (EntryMetricsHolder emh : instances.values()) { delay.addMetric(emh.destLabelValues, emh.latestExecTime.doubleValue()); + transactions.addMetric(emh.destLabelValues, emh.transactionCounter.doubleValue()); } mfs.add(delay); + mfs.add(transactions); return mfs; } @@ -51,25 +69,61 @@ public class EntryCollector extends Collector implements InstanceRegistry { throw new IllegalArgumentException("CanalEventSink must be EntryEventSink"); } EntryEventSink entrySink = (EntryEventSink) sink; - PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(); + PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink); holder.latestExecTime = handler.getLatestExecuteTime(); - entrySink.addHandler(handler, 0); - Preconditions.checkNotNull(holder.destLabelValues); + holder.transactionCounter = handler.getTransactionCounter(); Preconditions.checkNotNull(holder.latestExecTime); - EntryMetricsHolder old = instances.putIfAbsent(destination, holder); + Preconditions.checkNotNull(holder.transactionCounter); + EntryMetricsHolder old = instances.put(destination, holder); if (old != null) { - logger.warn("Ignore repeated EntryCollector register for instance {}.", destination); + logger.warn("Remove stale EntryCollector for instance {}.", destination); } } @Override public void unregister(CanalInstance instance) { final String destination = instance.getDestination(); + CanalEventSink sink = instance.getEventSink(); + if (!(sink instanceof EntryEventSink)) { + throw new IllegalArgumentException("CanalEventSink must be EntryEventSink"); + } + unloadHandler((EntryEventSink) sink); instances.remove(destination); } + private PrometheusCanalEventDownStreamHandler assembleHandler(EntryEventSink entrySink) { + PrometheusCanalEventDownStreamHandler ph = new PrometheusCanalEventDownStreamHandler(); + List handlers = entrySink.getHandlers(); + for (CanalEventDownStreamHandler handler : handlers) { + if (handler instanceof PrometheusCanalEventDownStreamHandler) { + throw new IllegalStateException("PrometheusCanalEventDownStreamHandler already exists in handlers."); + } + } + entrySink.addHandler(ph, 0); + return ph; + } + + private void unloadHandler(EntryEventSink entrySink) { + List handlers = entrySink.getHandlers(); + int i = 0; + for (; i < handlers.size(); i++) { + if (handlers.get(i) instanceof PrometheusCanalEventDownStreamHandler) { + break; + } + } + entrySink.removeHandler(i); + // Ensure no PrometheusCanalEventDownStreamHandler + handlers = entrySink.getHandlers(); + for (CanalEventDownStreamHandler handler : handlers) { + if (handler instanceof PrometheusCanalEventDownStreamHandler) { + throw new IllegalStateException("Multiple prometheusCanalEventDownStreamHandler exists in handlers."); + } + } + } + private class EntryMetricsHolder { private AtomicLong latestExecTime; + private AtomicLong transactionCounter; private List destLabelValues; } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java index eb5a2bde..aec9a424 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java @@ -23,18 +23,28 @@ import java.util.concurrent.ConcurrentMap; */ public class MetaCollector extends Collector implements InstanceRegistry { - private static final List InfoLabel = Arrays.asList("destination", "mode"); - private static final Logger logger = LoggerFactory.getLogger(MetaCollector.class); - private static final String SUBSCRIPTION = "canal_instance"; - private static final String subscriptionHelp = "Canal instance"; - private final ConcurrentMap instances = new ConcurrentHashMap(); + private static final List InfoLabel = Arrays.asList("destination", "mode"); + private static final Logger logger = LoggerFactory.getLogger(MetaCollector.class); + private static final String SUBSCRIPTION = "canal_instance"; + private static final String SUBSCRIPTION_HELP = "Canal instance"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + + private MetaCollector() {} + + private static class SingletonHolder { + private static final MetaCollector SINGLETON = new MetaCollector(); + } + + public static MetaCollector instance() { + return SingletonHolder.SINGLETON; + } @Override public List collect() { List mfs = new ArrayList(); GaugeMetricFamily instanceInfo = new GaugeMetricFamily( SUBSCRIPTION, - subscriptionHelp, + SUBSCRIPTION_HELP, InfoLabel); for (Map.Entry nme : instances.entrySet()) { final String destination = nme.getKey(); @@ -54,11 +64,10 @@ public class MetaCollector extends Collector implements InstanceRegistry { String mode = (instance instanceof CanalInstanceWithSpring) ? "spring" : "manager"; holder.infoLabelValues = Arrays.asList(destination, mode); holder.metaManager = instance.getMetaManager(); - Preconditions.checkNotNull(holder.infoLabelValues); Preconditions.checkNotNull(holder.metaManager); - MetaMetricsHolder old = instances.putIfAbsent(destination, holder); + MetaMetricsHolder old = instances.put(destination, holder); if (old != null) { - logger.warn("Ignore repeated MetaCollector register for instance {}.", destination); + logger.warn("Remove stale MetaCollector for instance {}.", destination); } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java deleted file mode 100644 index c8543f3c..00000000 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MysqlParserCollector.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.alibaba.otter.canal.prometheus.impl; - -import com.alibaba.otter.canal.common.utils.SerializedLongAdder; -import com.alibaba.otter.canal.parse.CanalEventParser; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; -import io.prometheus.client.Collector; -import io.prometheus.client.CounterMetricFamily; -import io.prometheus.client.GaugeMetricFamily; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST; -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; - -/** - * @author Chuanyi Li - */ -public class MysqlParserCollector extends Collector { - - private static final String PUBLISH_BLOCKING_TIME = "canal_instance_publish_blocking_time"; - private static final String RECEIVED_BINLOG_BYTES = "canal_instance_received_binlog_bytes"; - private static final String PARSER_MODE = "canal_instance_parser_mode"; - private static final String MODE_LABEL = "parallel"; - private final List destLabelValues; - private final List modeLabelValues; - private final SerializedLongAdder receivedBinlogBytes; - private final SerializedLongAdder eventsPublishBlockingTime; - private final String publishBlockingTimeHelp; - private final String receivedBinlogBytesHelp; - private final String modeHelp; - private final List modeLabels; - private final Boolean isParallel; - - public MysqlParserCollector(CanalEventParser parser, String destination) { - if (!(parser instanceof MysqlEventParser)) { - throw new IllegalArgumentException("CanalEventParser must be MysqlEventParser"); - } - this.destLabelValues = Collections.singletonList(destination); - this.eventsPublishBlockingTime = ((MysqlEventParser)parser).getEventsPublishBlockingTime(); - this.publishBlockingTimeHelp = "Blocking time of publishing of instance " + destination; - this.receivedBinlogBytes = ((MysqlEventParser)parser).getReceivedBinlogBytes(); - this.receivedBinlogBytesHelp = "Received binlog bytes of instance" + destination; - this.isParallel = ((MysqlEventParser)parser).isParallel(); - this.modeHelp = "Parser mode of instance" + destination; - this.modeLabels = Arrays.asList(DEST, MODE_LABEL); - this.modeLabelValues = Arrays.asList(destination, isParallel.toString()); - } - - @Override public List collect() { - List mfs = new ArrayList(); - // canal_instance_received_binlog_bytes - CounterMetricFamily bytesCounter = new CounterMetricFamily(RECEIVED_BINLOG_BYTES, - receivedBinlogBytesHelp, DEST_LABEL_LIST); - bytesCounter.addMetric(destLabelValues, receivedBinlogBytes.get()); - mfs.add(bytesCounter); - // canal_instance_parser_mode - GaugeMetricFamily modeGauge = new GaugeMetricFamily(PARSER_MODE, - modeHelp, - modeLabels); - modeGauge.addMetric(modeLabelValues, 1); - mfs.add(modeGauge); - // canal_instance_publish_blocking_time - if (isParallel) { - CounterMetricFamily blockingCounter = new CounterMetricFamily(PUBLISH_BLOCKING_TIME, - publishBlockingTimeHelp, DEST_LABEL_LIST); - blockingCounter.addMetric(destLabelValues, eventsPublishBlockingTime.get()); - mfs.add(blockingCounter); - } - return mfs; - } -} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java new file mode 100644 index 00000000..60906989 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java @@ -0,0 +1,114 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.parse.CanalEventParser; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; +import com.alibaba.otter.canal.prometheus.InstanceRegistry; +import com.google.common.base.Preconditions; +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import io.prometheus.client.GaugeMetricFamily; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; + +/** + * @author Chuanyi Li + */ +public class ParserCollector extends Collector implements InstanceRegistry { + + private static final Logger logger = LoggerFactory.getLogger(ParserCollector.class); + private static final String PUBLISH_BLOCKING = "canal_instance_publish_blocking_time"; + private static final String RECEIVED_BINLOG = "canal_instance_received_binlog_bytes"; + private static final String PARSER_MODE = "canal_instance_parser_mode"; + private static final String MODE_LABEL = "parallel"; + private static final String PUBLISH_BLOCKING_HELP = "Publish blocking time of dump thread"; + private static final String RECEIVED_BINLOG_HELP = "Received binlog bytes"; + private static final String MODE_HELP = "Parser mode(parallel/serial) of instance"; + private final List modeLabels = Arrays.asList(DEST, MODE_LABEL); + private final ConcurrentMap instances = new ConcurrentHashMap(); + + private ParserCollector() {} + + private static class SingletonHolder { + private static final ParserCollector SINGLETON = new ParserCollector(); + } + + public static ParserCollector instance() { + return SingletonHolder.SINGLETON; + } + + @Override + public List collect() { + List mfs = new ArrayList(); + boolean hasParallel = false; + CounterMetricFamily bytesCounter = new CounterMetricFamily(RECEIVED_BINLOG, + RECEIVED_BINLOG_HELP, DEST_LABELS_LIST); + GaugeMetricFamily modeGauge = new GaugeMetricFamily(PARSER_MODE, + MODE_HELP, modeLabels); + CounterMetricFamily blockingCounter = new CounterMetricFamily(PUBLISH_BLOCKING, + PUBLISH_BLOCKING_HELP, DEST_LABELS_LIST); + for (ParserMetricsHolder emh : instances.values()) { + if (emh.isParallel) { + blockingCounter.addMetric(emh.destLabelValues, emh.eventsPublishBlockingTime.doubleValue()); + hasParallel = true; + } + modeGauge.addMetric(emh.modeLabelValues, 1); + bytesCounter.addMetric(emh.destLabelValues, emh.receivedBinlogBytes.doubleValue()); + + } + mfs.add(bytesCounter); + mfs.add(modeGauge); + if (hasParallel) { + mfs.add(blockingCounter); + } + return mfs; + } + + @Override + public void register(CanalInstance instance) { + final String destination = instance.getDestination(); + ParserMetricsHolder holder = new ParserMetricsHolder(); + CanalEventParser parser = instance.getEventParser(); + if (!(parser instanceof MysqlEventParser)) { + throw new IllegalArgumentException("CanalEventParser must be MysqlEventParser"); + } + MysqlEventParser mysqlParser = (MysqlEventParser) parser; + holder.destLabelValues = Collections.singletonList(destination); + holder.modeLabelValues = Arrays.asList(destination, Boolean.toString(mysqlParser.isParallel())); + holder.eventsPublishBlockingTime = mysqlParser.getEventsPublishBlockingTime(); + holder.receivedBinlogBytes = mysqlParser.getReceivedBinlogBytes(); + holder.isParallel = mysqlParser.isParallel(); + Preconditions.checkNotNull(holder.eventsPublishBlockingTime); + Preconditions.checkNotNull(holder.receivedBinlogBytes); + ParserMetricsHolder old = instances.put(destination, holder); + if (old != null) { + logger.warn("Remove stale ParserCollector for instance {}.", destination); + } + } + + @Override + public void unregister(CanalInstance instance) { + final String destination = instance.getDestination(); + instances.remove(destination); + } + + private class ParserMetricsHolder { + private List destLabelValues; + private List modeLabelValues; + private AtomicLong receivedBinlogBytes; + private AtomicLong eventsPublishBlockingTime; + private boolean isParallel; + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java index 763ce46b..6fc2f3d7 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java @@ -1,7 +1,9 @@ package com.alibaba.otter.canal.prometheus.impl; +import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler; import com.alibaba.otter.canal.store.model.Event; + import java.util.List; import java.util.concurrent.atomic.AtomicLong; @@ -10,28 +12,71 @@ import java.util.concurrent.atomic.AtomicLong; */ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler> { - private final AtomicLong latestExecuteTime = new AtomicLong(0L); + private final AtomicLong latestExecuteTime = new AtomicLong(0L); + private final AtomicLong transactionCounter = new AtomicLong(0L); + private final AtomicLong rowEventCounter = new AtomicLong(0L); + private final AtomicLong rowsCounter = new AtomicLong(0L); @Override public List before(List events) { - // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming - // see: https://dev.mysql.com/worklog/task/?id=342 - // heartbeats are sent by the master only if there is no - // more unsent events in the actual binlog file for a period longer that - // master_heartbeat_period. + long localExecTime = 0L; if (events != null && !events.isEmpty()) { - Event last = events.get(events.size() - 1); - long ts = last.getExecuteTime(); - long ls = latestExecuteTime.get(); - if (ts > ls) { - latestExecuteTime.lazySet(ts); + for (Event e : events) { + EntryType type = e.getEntryType(); + if (type == null) continue; + long exec = e.getExecuteTime(); + if (exec > 0) localExecTime = exec; + switch (type) { + case ROWDATA: + // TODO 当前proto无法直接获得荣威change的变更行数（需要parse），可考虑放到header里面 + break; + case TRANSACTIONEND: + transactionCounter.incrementAndGet(); + break; + case HEARTBEAT: + // TODO 确认一下不是canal自己产生的 + // EventType eventType = e.getEventType(); + // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming + // see: https://dev.mysql.com/worklog/task/?id=342 + // heartbeats are sent by the master only if there is no + // more unsent events in the actual binlog file for a period longer that + // master_heartbeat_period. + break; + default: + break; + } + } + if (localExecTime > 0) { + latestExecuteTime.lazySet(localExecTime); } } return events; } + @Override + public void start() { + + super.start(); + } + + @Override + public void stop() { + super.stop(); + } + public AtomicLong getLatestExecuteTime() { return latestExecuteTime; } + public AtomicLong getTransactionCounter() { + return transactionCounter; + } + + public AtomicLong getRowsCounter() { + return rowsCounter; + } + + public AtomicLong getRowEventCounter() { + return rowEventCounter; + } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java index 11903314..fc5ed3b2 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java @@ -17,23 +17,33 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; -import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL_LIST; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; /** * @author Chuanyi Li */ public class SinkCollector extends Collector implements InstanceRegistry { - private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); - private static final String SINK_BLOCKING_TIME = "canal_instance_sink_blocking_time"; - private static final String sinkBlockTimeHelp = "Total sink blocking time"; - private final ConcurrentMap instances = new ConcurrentHashMap(); + private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); + private static final String SINK_BLOCKING_TIME = "canal_instance_sink_blocking_time"; + private static final String SINK_BLOCK_TIME_HELP = "Total sink blocking time"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + + private SinkCollector() {} + + private static class SingletonHolder { + private static final SinkCollector SINGLETON = new SinkCollector(); + } + + public static SinkCollector instance() { + return SingletonHolder.SINGLETON; + } @Override public List collect() { List mfs = new ArrayList(); CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME, - sinkBlockTimeHelp, DEST_LABEL_LIST); + SINK_BLOCK_TIME_HELP, DEST_LABELS_LIST); for (SinkMetricsHolder smh : instances.values()) { blockingCounter.addMetric(smh.destLabelValues, smh.eventsSinkBlockingTime.doubleValue()); } @@ -52,11 +62,10 @@ public class SinkCollector extends Collector implements InstanceRegistry { } EntryEventSink entrySink = (EntryEventSink) sink; holder.eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime(); - Preconditions.checkNotNull(holder.destLabelValues); Preconditions.checkNotNull(holder.eventsSinkBlockingTime); - SinkMetricsHolder old = instances.putIfAbsent(destination, holder); + SinkMetricsHolder old = instances.put(destination, holder); if (old != null) { - logger.warn("Ignore repeated SinkCollector register for instance {}.", destination); + logger.warn("Remote stale SinkCollector for instance {}.", destination); } } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java index 3784af9b..8bd39574 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java @@ -1,15 +1,17 @@ package com.alibaba.otter.canal.prometheus.impl; import com.alibaba.otter.canal.instance.core.CanalInstance; -import com.alibaba.otter.canal.prometheus.CanalInstanceExports; import com.alibaba.otter.canal.prometheus.InstanceRegistry; import com.alibaba.otter.canal.store.CanalEventStore; -import com.alibaba.otter.canal.store.CanalStoreException; import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; +import com.alibaba.otter.canal.store.model.BatchMode; +import com.google.common.base.Preconditions; import io.prometheus.client.Collector; import io.prometheus.client.CounterMetricFamily; +import io.prometheus.client.GaugeMetricFamily; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -18,68 +20,113 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; + /** * @author Chuanyi Li */ public class StoreCollector extends Collector implements InstanceRegistry { - private static final Class clazz = MemoryEventStoreWithBuffer.class; - private static final String PRODUCE = "canal_instance_store_produce_seq"; - private static final String CONSUME = "canal_instance_store_consume_seq"; - private static final String produceHelp = "Produced sequence of canal instance"; - private static final String consumeHelp = "Consumed sequence of canal instance"; - private final ConcurrentMap instances = new ConcurrentHashMap(); + private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); + private static final String PRODUCE = "canal_instance_store_produce_seq"; + private static final String CONSUME = "canal_instance_store_consume_seq"; + private static final String STORE = "canal_instance_store"; + private static final String PRODUCE_MEM = "canal_instance_store_produce_mem"; + private static final String CONSUME_MEM = "canal_instance_store_consume_mem"; + private static final String PRODUCE_HELP = "Produced events counter of canal instance"; + private static final String CONSUME_HELP = "Consumed events counter of canal instance"; + private static final String STORE_HELP = "Canal instance info"; + private static final String PRODUCE_MEM_HELP = "Produced mem bytes of canal instance"; + private static final String CONSUME_MEM_HELP = "Consumed mem bytes of canal instance"; + private final ConcurrentMap instances = new ConcurrentHashMap(); + private final List storeLabelsList = Arrays.asList(DEST, "batchMode"); + private StoreCollector() {} - public StoreCollector(CanalEventStore store, String destination) { - this.destination = destination; - if (!(store instanceof MemoryEventStoreWithBuffer)) { - throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer"); - } - MemoryEventStoreWithBuffer ms = (MemoryEventStoreWithBuffer) store; - putSequence = getDeclaredValue(ms, "putSequence"); - ackSequence = getDeclaredValue(ms, "ackSequence"); + private static class SingletonHolder { + private static final StoreCollector SINGLETON = new StoreCollector(); + } + + public static StoreCollector instance() { + return SingletonHolder.SINGLETON; } @Override public List collect() { List mfs = new ArrayList(); - CounterMetricFamily put = new CounterMetricFamily("canal_instance_store_produce_seq", - putHelp, Arrays.asList(CanalInstanceExports.DEST_LABELS)); - put.addMetric(Collections.singletonList(destination), putSequence.doubleValue()); + CounterMetricFamily put = new CounterMetricFamily(PRODUCE, + PRODUCE_HELP, DEST_LABELS_LIST); + CounterMetricFamily ack = new CounterMetricFamily(CONSUME, + CONSUME_HELP, DEST_LABELS_LIST); + GaugeMetricFamily store = new GaugeMetricFamily(STORE, + STORE_HELP, storeLabelsList); + CounterMetricFamily putMem = new CounterMetricFamily(PRODUCE_MEM, + PRODUCE_MEM_HELP, DEST_LABELS_LIST); + CounterMetricFamily ackMem = new CounterMetricFamily(CONSUME_MEM, + CONSUME_MEM_HELP, DEST_LABELS_LIST); + boolean hasMem = false; + for (StoreMetricsHolder smh : instances.values()) { + final boolean isMem = smh.batchMode.isMemSize(); + put.addMetric(smh.destLabelValues, smh.putSeq.doubleValue()); + ack.addMetric(smh.destLabelValues, smh.ackSeq.doubleValue()); + store.addMetric(smh.storeLabelValues, 1); + if (isMem) { + hasMem = true; + putMem.addMetric(smh.destLabelValues, smh.putMemSize.doubleValue()); + ackMem.addMetric(smh.destLabelValues, smh.ackMemSize.doubleValue()); + } + } mfs.add(put); - CounterMetricFamily ack = new CounterMetricFamily("canal_instance_store_consume_seq", - ackHelp, Arrays.asList(CanalInstanceExports.DEST_LABELS)); - ack.addMetric(Collections.singletonList(destination), ackSequence.doubleValue()); mfs.add(ack); + mfs.add(store); + if (hasMem) { + mfs.add(putMem); + mfs.add(ackMem); + } return mfs; } - @SuppressWarnings("unchecked") - private static T getDeclaredValue(MemoryEventStoreWithBuffer store, String name) { - T value; - try { - Field putField = clazz.getDeclaredField(name); - putField.setAccessible(true); - value = (T) putField.get(store); - } catch (NoSuchFieldException e) { - throw new CanalStoreException(e); - } catch (IllegalAccessException e) { - throw new CanalStoreException(e); - } - return value; - } - @Override public void register(CanalInstance instance) { - + final String destination = instance.getDestination(); + StoreMetricsHolder holder = new StoreMetricsHolder(); + CanalEventStore store = instance.getEventStore(); + if (!(store instanceof MemoryEventStoreWithBuffer)) { + throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer"); + } + MemoryEventStoreWithBuffer memStore = (MemoryEventStoreWithBuffer) store; + holder.batchMode = memStore.getBatchMode(); + holder.putSeq = memStore.getPutSequence(); + holder.ackSeq = memStore.getAckSequence(); + holder.destLabelValues = Collections.singletonList(destination); + holder.storeLabelValues = Arrays.asList(destination, memStore.getBatchMode().name()); + Preconditions.checkNotNull(holder.batchMode); + Preconditions.checkNotNull(holder.putSeq); + Preconditions.checkNotNull(holder.ackSeq); + if (holder.batchMode.isMemSize()) { + holder.putMemSize = memStore.getPutMemSize(); + holder.ackMemSize = memStore.getAckMemSize(); + Preconditions.checkNotNull(holder.putMemSize); + Preconditions.checkNotNull(holder.ackMemSize); + } + StoreMetricsHolder old = instances.putIfAbsent(destination, holder); + if (old != null) { + logger.warn("Remote stale StoreCollector for instance {}.", destination); + } } @Override public void unregister(CanalInstance instance) { - + final String destination = instance.getDestination(); + instances.remove(destination); } private class StoreMetricsHolder { - private AtomicLong putSeq; - private AtomicLong ackSeq; + private AtomicLong putSeq; + private AtomicLong ackSeq; + private BatchMode batchMode; + private AtomicLong putMemSize; + private AtomicLong ackMemSize; + private List destLabelValues; + private List storeLabelValues; } } diff --git a/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java b/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java index 075d1524..00ba2e71 100644 --- a/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java +++ b/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java @@ -562,4 +562,23 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple this.ddlIsolation = ddlIsolation; } + public AtomicLong getPutSequence() { + return putSequence; + } + + public AtomicLong getAckSequence() { + return ackSequence; + } + + public AtomicLong getPutMemSize() { + return putMemSize; + } + + public AtomicLong getAckMemSize() { + return ackMemSize; + } + + public BatchMode getBatchMode() { + return batchMode; + } } From d24e10fb3e0ef9e338859026c5e4262d761c26a1 Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Tue, 7 Aug 2018 22:52:30 +0800 Subject: [PATCH 21/21] Canal metrics snapshot version. --- deployer/pom.xml | 12 ++++---- deployer/src/main/bin/metrics_env.sh | 15 ---------- deployer/src/main/bin/startup.sh | 7 +---- .../parse/inbound/mysql/MysqlConnection.java | 3 +- .../mysql/MysqlMultiStageCoprocessor.java | 5 ++++ pom.xml | 2 +- .../prometheus/CanalInstanceExports.java | 2 +- .../canal/prometheus/PrometheusService.java | 11 +++---- .../canal/prometheus/impl/EntryCollector.java | 8 +++-- .../canal/prometheus/impl/MetaCollector.java | 29 +++++++++++-------- .../prometheus/impl/ParserCollector.java | 6 ++-- ...PrometheusCanalEventDownStreamHandler.java | 21 ++++++++++---- .../PrometheusClientInstanceProfiler.java | 15 ++++++++-- .../canal/prometheus/impl/SinkCollector.java | 6 ++-- .../canal/sink/entry/EntryEventSink.java | 5 ++++ 15 files changed, 86 insertions(+), 61 deletions(-) delete mode 100644 deployer/src/main/bin/metrics_env.sh diff --git a/deployer/pom.xml b/deployer/pom.xml index 75f05805..68224ca1 100644 --- a/deployer/pom.xml +++ b/deployer/pom.xml @@ -18,12 +18,12 @@ - - - - - - + + com.alibaba.otter + canal.prometheus + ${project.version} + runtime + diff --git a/deployer/src/main/bin/metrics_env.sh b/deployer/src/main/bin/metrics_env.sh deleted file mode 100644 index 1085eae2..00000000 --- a/deployer/src/main/bin/metrics_env.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Additional line arg for current prometheus solution -case "`uname`" in -Linux) - bin_abs_path=$(readlink -f $(dirname $0)) - ;; -*) - bin_abs_path=`cd $(dirname $0); pwd` - ;; -esac -base=${bin_abs_path}/.. -if [ $(ls $base/lib/aspectjweaver*.jar | wc -l) -eq 1 ]; then - WEAVER=$(ls $base/lib/aspectjweaver*.jar) - METRICS_OPTS=" -javaagent:"${WEAVER}" " -fi diff --git a/deployer/src/main/bin/startup.sh b/deployer/src/main/bin/startup.sh index b87652c0..5655c713 100644 --- a/deployer/src/main/bin/startup.sh +++ b/deployer/src/main/bin/startup.sh @@ -94,12 +94,7 @@ then echo LOG CONFIGURATION : $logback_configurationFile echo canal conf : $canal_conf echo CLASSPATH :$CLASSPATH -# metrics support options -# if [ -x $base/bin/metrics_env.sh ]; then -# . $base/bin/metrics_env.sh -# echo METRICS_OPTS $METRICS_OPTS -# fi - $JAVA $JAVA_OPTS $METRICS_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & + $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & echo $! > $base/bin/canal.pid echo "cd to $current_path for continue" diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 9d89a145..5d2dbb4d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -50,7 +50,8 @@ public class MysqlConnection implements ErosaConnection { private AuthenticationInfo authInfo; protected int connTimeout = 5 * 1000; // 5秒 protected int soTimeout = 60 * 60 * 1000; // 1小时 - private AtomicLong receivedBinlogBytes; + // dump binlog bytes, 暂不包括meta与TSDB + private AtomicLong receivedBinlogBytes; public MysqlConnection(){ } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java index 92447d50..4fef7710 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java @@ -187,6 +187,11 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement //LockSupport.parkNanos(1L); applyWait(++fullTimes); interupted = Thread.interrupted(); + if (fullTimes % 1000 == 0) { + long nextStart = System.nanoTime(); + eventsPublishBlockingTime.addAndGet(nextStart - blockingStart); + blockingStart = nextStart; + } } } while (!interupted && isStart()); return isStart(); diff --git a/pom.xml b/pom.xml index 46d103e3..9a14868b 100644 --- a/pom.xml +++ b/pom.xml @@ -247,7 +247,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_186 + 2.0.0_preview_520 com.alibaba diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java index 487855fa..6f50214c 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -77,7 +77,7 @@ public class CanalInstanceExports { private InstanceRegistry requiredInstanceRegistry(Collector collector) { if (!(collector instanceof InstanceRegistry)) { - throw new IllegalArgumentException("Canal collector need to implement InstanceRegistry."); + throw new IllegalArgumentException("Canal prometheus collector need to implement InstanceRegistry."); } return (InstanceRegistry) collector; } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java index 9e3ef18c..485f6728 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -19,14 +19,15 @@ import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler. */ public class PrometheusService implements CanalMetricsService { - private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); - private final CanalInstanceExports instanceExports; - private volatile boolean running = false; - private HTTPServer server; - private ClientInstanceProfiler clientProfiler = new PrometheusClientInstanceProfiler(); + private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); + private final CanalInstanceExports instanceExports; + private volatile boolean running = false; + private HTTPServer server; + private final ClientInstanceProfiler clientProfiler; private PrometheusService() { this.instanceExports = CanalInstanceExports.instance(); + this.clientProfiler = PrometheusClientInstanceProfiler.instance(); } private static class SingletonHolder { diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java index dd282065..4aa6ff42 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java @@ -29,7 +29,7 @@ public class EntryCollector extends Collector implements InstanceRegistry { private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); private static final String DELAY = "canal_instance_traffic_delay"; private static final String TRANSACTION = "canal_instance_transactions"; - private static final String DELAY_HELP = "Traffic delay of canal instance"; + private static final String DELAY_HELP = "Traffic delay of canal instance in milliseconds"; private static final String TRANSACTION_HELP = "Transactions counter of canal instance"; private final ConcurrentMap instances = new ConcurrentHashMap(); @@ -51,7 +51,11 @@ public class EntryCollector extends Collector implements InstanceRegistry { CounterMetricFamily transactions = new CounterMetricFamily(TRANSACTION, TRANSACTION_HELP, DEST_LABELS_LIST); for (EntryMetricsHolder emh : instances.values()) { - delay.addMetric(emh.destLabelValues, emh.latestExecTime.doubleValue()); + long now = System.currentTimeMillis(); + long latest = emh.latestExecTime.get(); + if (now > latest) { + delay.addMetric(emh.destLabelValues, (now - latest)); + } transactions.addMetric(emh.destLabelValues, emh.transactionCounter.doubleValue()); } mfs.add(delay); diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java index aec9a424..e51e8e29 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MetaCollector.java @@ -11,22 +11,23 @@ import io.prometheus.client.GaugeMetricFamily; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; + /** * @author Chuanyi Li */ public class MetaCollector extends Collector implements InstanceRegistry { - private static final List InfoLabel = Arrays.asList("destination", "mode"); + private static final List INFO_LABELS_LIST = Arrays.asList("destination", "mode"); private static final Logger logger = LoggerFactory.getLogger(MetaCollector.class); - private static final String SUBSCRIPTION = "canal_instance"; - private static final String SUBSCRIPTION_HELP = "Canal instance"; + private static final String INSTANCE = "canal_instance"; + private static final String INSTANCE_HELP = "Canal instance"; + private static final String SUBSCRIPTION = "canal_instance_subscriptions"; + private static final String SUBSCRIPTION_HELP = "Canal instance subscriptions"; private final ConcurrentMap instances = new ConcurrentHashMap(); private MetaCollector() {} @@ -42,18 +43,20 @@ public class MetaCollector extends Collector implements InstanceRegistry { @Override public List collect() { List mfs = new ArrayList(); - GaugeMetricFamily instanceInfo = new GaugeMetricFamily( - SUBSCRIPTION, - SUBSCRIPTION_HELP, - InfoLabel); + GaugeMetricFamily instanceInfo = new GaugeMetricFamily(INSTANCE, + INSTANCE_HELP, INFO_LABELS_LIST); + GaugeMetricFamily subsInfo = new GaugeMetricFamily(SUBSCRIPTION, + SUBSCRIPTION_HELP, DEST_LABELS_LIST); for (Map.Entry nme : instances.entrySet()) { final String destination = nme.getKey(); final MetaMetricsHolder nmh = nme.getValue(); + instanceInfo.addMetric(nmh.infoLabelValues, 1); List subs = nmh.metaManager.listAllSubscribeInfo(destination); int count = subs == null ? 0 : subs.size(); - instanceInfo.addMetric(nmh.infoLabelValues, count); + subsInfo.addMetric(nmh.destLabelValues, count); } mfs.add(instanceInfo); + mfs.add(subsInfo); return mfs; } @@ -63,6 +66,7 @@ public class MetaCollector extends Collector implements InstanceRegistry { MetaMetricsHolder holder = new MetaMetricsHolder(); String mode = (instance instanceof CanalInstanceWithSpring) ? "spring" : "manager"; holder.infoLabelValues = Arrays.asList(destination, mode); + holder.destLabelValues = Collections.singletonList(destination); holder.metaManager = instance.getMetaManager(); Preconditions.checkNotNull(holder.metaManager); MetaMetricsHolder old = instances.put(destination, holder); @@ -79,6 +83,7 @@ public class MetaCollector extends Collector implements InstanceRegistry { private class MetaMetricsHolder { private List infoLabelValues; + private List destLabelValues; private CanalMetaManager metaManager; } diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java index 60906989..73102795 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicLong; import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST; import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; +import static java.util.concurrent.TimeUnit.NANOSECONDS; /** * @author Chuanyi Li @@ -28,11 +29,12 @@ import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL public class ParserCollector extends Collector implements InstanceRegistry { private static final Logger logger = LoggerFactory.getLogger(ParserCollector.class); + private static final long NANO_PER_MILLI = 1000 * 1000L; private static final String PUBLISH_BLOCKING = "canal_instance_publish_blocking_time"; private static final String RECEIVED_BINLOG = "canal_instance_received_binlog_bytes"; private static final String PARSER_MODE = "canal_instance_parser_mode"; private static final String MODE_LABEL = "parallel"; - private static final String PUBLISH_BLOCKING_HELP = "Publish blocking time of dump thread"; + private static final String PUBLISH_BLOCKING_HELP = "Publish blocking time of dump thread in milliseconds"; private static final String RECEIVED_BINLOG_HELP = "Received binlog bytes"; private static final String MODE_HELP = "Parser mode(parallel/serial) of instance"; private final List modeLabels = Arrays.asList(DEST, MODE_LABEL); @@ -60,7 +62,7 @@ public class ParserCollector extends Collector implements InstanceRegistry { PUBLISH_BLOCKING_HELP, DEST_LABELS_LIST); for (ParserMetricsHolder emh : instances.values()) { if (emh.isParallel) { - blockingCounter.addMetric(emh.destLabelValues, emh.eventsPublishBlockingTime.doubleValue()); + blockingCounter.addMetric(emh.destLabelValues, (emh.eventsPublishBlockingTime.doubleValue() / NANO_PER_MILLI)); hasParallel = true; } modeGauge.addMetric(emh.modeLabelValues, 1); diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java index 6fc2f3d7..ebbbfa00 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java @@ -1,5 +1,6 @@ package com.alibaba.otter.canal.prometheus.impl; +import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler; import com.alibaba.otter.canal.store.model.Event; @@ -24,18 +25,28 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow for (Event e : events) { EntryType type = e.getEntryType(); if (type == null) continue; - long exec = e.getExecuteTime(); - if (exec > 0) localExecTime = exec; switch (type) { - case ROWDATA: + case TRANSACTIONBEGIN: { + long exec = e.getExecuteTime(); + if (exec > 0) localExecTime = exec; + break; + } + case ROWDATA: { + long exec = e.getExecuteTime(); + if (exec > 0) localExecTime = exec; // TODO 当前proto无法直接获得荣威change的变更行数（需要parse），可考虑放到header里面 break; - case TRANSACTIONEND: + } + case TRANSACTIONEND: { + long exec = e.getExecuteTime(); + if (exec > 0) localExecTime = exec; transactionCounter.incrementAndGet(); break; + } case HEARTBEAT: + // 发现canal自己的heartbeat是带有execTime的 // TODO 确认一下不是canal自己产生的 - // EventType eventType = e.getEventType(); + CanalEntry.EventType eventType = e.getEventType(); // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming // see: https://dev.mysql.com/worklog/task/?id=342 // heartbeats are sent by the master only if there is no diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java index 331897ee..da16f2db 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java @@ -15,6 +15,7 @@ import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL */ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler { + private static final long NANO_PER_MILLI = 1000 * 1000L; private static final String PACKET_TYPE = "canal_instance_client_packets"; private static final String OUTBOUND_BYTES = "canal_instance_client_bytes"; private static final String EMPTY_BATCHES = "canal_instance_client_empty_batches"; @@ -27,7 +28,15 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler private final Histogram responseLatency; private volatile boolean running = false; - public PrometheusClientInstanceProfiler() { + private static class SingletonHolder { + private static final PrometheusClientInstanceProfiler SINGLETON = new PrometheusClientInstanceProfiler(); + } + + public static PrometheusClientInstanceProfiler instance() { + return SingletonHolder.SINGLETON; + } + + private PrometheusClientInstanceProfiler() { this.outboundCounter = Counter.build() .labelNames(DEST_LABELS) .name(OUTBOUND_BYTES) @@ -53,7 +62,7 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler .name(LATENCY) .help("Client request latency.") // buckets in milliseconds - .buckets(1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0) + .buckets(2.5, 10.0, 25.0, 100.0) .create(); } @@ -68,7 +77,7 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler errorsCounter.labels(destination, Short.toString(errorCode)).inc(); } long latency = result.getLatency(); - responseLatency.labels(destination).observe(latency / 1000000); + responseLatency.labels(destination).observe(((double) latency) / NANO_PER_MILLI); switch (type) { case GET: boolean empty = result.getEmpty(); diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java index fc5ed3b2..61dc4d64 100644 --- a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java @@ -18,6 +18,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; +import static java.util.concurrent.TimeUnit.NANOSECONDS; /** * @author Chuanyi Li @@ -25,8 +26,9 @@ import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABEL public class SinkCollector extends Collector implements InstanceRegistry { private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class); + private static final long NANO_PER_MILLI = 1000 * 1000L; private static final String SINK_BLOCKING_TIME = "canal_instance_sink_blocking_time"; - private static final String SINK_BLOCK_TIME_HELP = "Total sink blocking time"; + private static final String SINK_BLOCK_TIME_HELP = "Total sink blocking time in milliseconds"; private final ConcurrentMap instances = new ConcurrentHashMap(); private SinkCollector() {} @@ -45,7 +47,7 @@ public class SinkCollector extends Collector implements InstanceRegistry { CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME, SINK_BLOCK_TIME_HELP, DEST_LABELS_LIST); for (SinkMetricsHolder smh : instances.values()) { - blockingCounter.addMetric(smh.destLabelValues, smh.eventsSinkBlockingTime.doubleValue()); + blockingCounter.addMetric(smh.destLabelValues, (smh.eventsSinkBlockingTime.doubleValue() / NANO_PER_MILLI)); } mfs.add(blockingCounter); return mfs; diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java index 23c5071b..699b06a5 100644 --- a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java +++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java @@ -164,6 +164,11 @@ public class EntryEventSink extends AbstractCanalEventSink> handler : getHandlers()) {