diff --git a/deployer/pom.xml b/deployer/pom.xml index 5fa2efc8..75f05805 100644 --- a/deployer/pom.xml +++ b/deployer/pom.xml @@ -16,6 +16,14 @@ canal.server ${project.version} + + + + + + + + diff --git a/deployer/src/main/bin/metrics_env.sh b/deployer/src/main/bin/metrics_env.sh new file mode 100644 index 00000000..1085eae2 --- /dev/null +++ b/deployer/src/main/bin/metrics_env.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Additional line arg for current prometheus solution +case "`uname`" in +Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; +*) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +if [ $(ls $base/lib/aspectjweaver*.jar | wc -l) -eq 1 ]; then + WEAVER=$(ls $base/lib/aspectjweaver*.jar) + METRICS_OPTS=" -javaagent:"${WEAVER}" " +fi diff --git a/deployer/src/main/bin/startup.sh b/deployer/src/main/bin/startup.sh index 5655c713..b87652c0 100644 --- a/deployer/src/main/bin/startup.sh +++ b/deployer/src/main/bin/startup.sh @@ -94,7 +94,12 @@ then echo LOG CONFIGURATION : $logback_configurationFile echo canal conf : $canal_conf echo CLASSPATH :$CLASSPATH - $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & +# metrics support options +# if [ -x $base/bin/metrics_env.sh ]; then +# . $base/bin/metrics_env.sh +# echo METRICS_OPTS $METRICS_OPTS +# fi + $JAVA $JAVA_OPTS $METRICS_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & echo $! > $base/bin/canal.pid echo "cd to $current_path for continue" diff --git a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java index 1b84463b..36979177 100644 --- a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java +++ b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java @@ -3,6 +3,7 @@ package com.alibaba.otter.canal.example; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.SystemUtils; @@ -115,7 +116,7 @@ public class AbstractCanalClientTest { connector.connect(); connector.subscribe(); while (running) { - Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 + Message message = connector.getWithoutAck(batchSize, 1000L, TimeUnit.MILLISECONDS); // 获取指定数量的数据 long batchId = message.getId(); int size = message.getEntries().size(); if (batchId == -1 || size == 0) { diff --git a/pom.xml b/pom.xml index 1e8123ef..8b824ab5 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,7 @@ example kafka kafka-client + prometheus @@ -256,7 +257,7 @@ com.alibaba.fastsql fastsql - 2.0.0_preview_520 + 2.0.0_preview_186 com.alibaba diff --git a/prometheus/pom.xml b/prometheus/pom.xml new file mode 100644 index 00000000..16cf8acf --- /dev/null +++ b/prometheus/pom.xml @@ -0,0 +1,67 @@ + + + + canal + com.alibaba.otter + 1.0.26-SNAPSHOT + + 4.0.0 + + com.alibaba.otter + canal.prometheus + 1.0.26-SNAPSHOT + canal prometheus module for otter ${project.version} + + + + org.aspectj + aspectjrt + 1.8.9 + + + org.aspectj + aspectjweaver + 1.8.9 + + + org.jctools + jctools-core + 2.1.2 + + + io.prometheus + simpleclient + 0.4.0 + + + io.prometheus + simpleclient_hotspot + 0.4.0 + + + io.prometheus + simpleclient_httpserver + 0.4.0 + + + io.prometheus + simpleclient_pushgateway + 0.4.0 + + + com.alibaba.otter + canal.instance.core + ${project.version} + provided + + + com.alibaba.otter + canal.server + ${project.version} + provided + + + + diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java new file mode 100644 index 00000000..8acf40b8 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java @@ -0,0 +1,98 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.prometheus.impl.InstanceMetaCollector; +import com.alibaba.otter.canal.prometheus.impl.MemoryStoreCollector; +import com.alibaba.otter.canal.prometheus.impl.PrometheusCanalEventDownStreamHandler; +import com.alibaba.otter.canal.sink.CanalEventSink; +import com.alibaba.otter.canal.sink.entry.EntryEventSink; +import com.alibaba.otter.canal.store.CanalStoreException; +import io.prometheus.client.Collector; +import io.prometheus.client.CollectorRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; + +/** + * @author Chuanyi Li + */ +public class CanalInstanceExports { + + private static final Logger logger = LoggerFactory.getLogger(CanalInstanceExports.class); + + public static final String[] labels = {"destination"}; + + public static final List labelList = Collections.singletonList(labels[0]); + + private final String destination; + + private Collector storeCollector; + + private Collector delayCollector; + + private Collector metaCollector; + + private CanalInstanceExports(CanalInstance instance) { + this.destination = instance.getDestination(); + initDelayGauge(instance); + initStoreCollector(instance); + initMetaCollector(instance); + } + + + + static CanalInstanceExports forInstance(CanalInstance instance) { + return new CanalInstanceExports(instance); + } + + void register() { + if (delayCollector != null) { + delayCollector.register(); + } + if (storeCollector != null) { + storeCollector.register(); + } + if (metaCollector != null) { + metaCollector.register(); + } + } + + void unregister() { + if (delayCollector != null) { + CollectorRegistry.defaultRegistry.unregister(delayCollector); + } + if (storeCollector != null) { + CollectorRegistry.defaultRegistry.unregister(storeCollector); + } + if (metaCollector != null) { + CollectorRegistry.defaultRegistry.unregister(metaCollector); + } + } + + private void initDelayGauge(CanalInstance instance) { + CanalEventSink sink = instance.getEventSink(); + if (sink instanceof EntryEventSink) { + EntryEventSink entryEventSink = (EntryEventSink) sink; + // TODO ensure not to add handler again + PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination); + entryEventSink.addHandler(handler); + delayCollector = handler.getCollector(); + } else { + logger.warn("This impl register metrics for only EntryEventSink, skip."); + } + } + + private void initStoreCollector(CanalInstance instance) { + try { + storeCollector = new MemoryStoreCollector(instance.getEventStore(), destination); + } catch (CanalStoreException cse) { + logger.warn("Failed to register metrics for destination {}.", destination, cse); + } + } + + private void initMetaCollector(CanalInstance instance) { + metaCollector = new InstanceMetaCollector(instance); + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java new file mode 100644 index 00000000..8eef79ba --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java @@ -0,0 +1,21 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.prometheus.impl.InboundThroughputAspect; +import com.alibaba.otter.canal.prometheus.impl.OutboundThroughputAspect; + +/** + * @author Chuanyi Li + */ +public class CanalServerExports { + + private static boolean initialized = false; + + public static synchronized void initialize() { + if (!initialized) { + InboundThroughputAspect.getCollector().register(); + OutboundThroughputAspect.getCollector().register(); + initialized = true; + } + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java new file mode 100644 index 00000000..12b8675f --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java @@ -0,0 +1,15 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.spi.CanalMetricsProvider; +import com.alibaba.otter.canal.spi.CanalMetricsService; + +/** + * @author Chuanyi Li + */ +public class PrometheusProvider implements CanalMetricsProvider { + + @Override + public CanalMetricsService getService() { + return PrometheusService.getInstance(); + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java new file mode 100644 index 00000000..c7942235 --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java @@ -0,0 +1,110 @@ +package com.alibaba.otter.canal.prometheus; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.spi.CanalMetricsService; +import io.prometheus.client.exporter.HTTPServer; +import io.prometheus.client.hotspot.DefaultExports; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author Chuanyi Li + */ +public class PrometheusService implements CanalMetricsService { + + private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); + + private final Map exports = new ConcurrentHashMap(); + + private volatile boolean running = false; + + private HTTPServer server; + + private PrometheusService() { + } + + private static class SingletonHolder { + private static final PrometheusService SINGLETON = new PrometheusService(); + } + + public static PrometheusService getInstance() { + return SingletonHolder.SINGLETON; + } + + @Override + public void initialize() { + try { + //TODO 1.Configurable port + //TODO 2.Https + server = new HTTPServer(11112); + } catch (IOException e) { + logger.warn("Unable to start prometheus HTTPServer.", e); + return; + } + try { + // JVM exports + DefaultExports.initialize(); + // Canal server level exports + CanalServerExports.initialize(); + } catch (Throwable t) { + logger.warn("Unable to initialize server exports.", t); + } + + running = true; + } + + @Override + public void terminate() { + running = false; + // Normally, service should be terminated at canal shutdown. + // No need to unregister instance exports explicitly. + // But for the sake of safety, unregister them. + for (CanalInstanceExports ie : exports.values()) { + ie.unregister(); + } + if (server != null) { + server.stop(); + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void register(CanalInstance instance) { + if (instance.isStart()) { + logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination()); + return; + } + try { + CanalInstanceExports export = CanalInstanceExports.forInstance(instance); + export.register(); + exports.put(instance.getDestination(), export); + } catch (Throwable t) { + logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t); + } + logger.info("Register metrics for destination {}.", instance.getDestination()); + } + + @Override + public void unregister(CanalInstance instance) { + if (instance.isStart()) { + logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination()); + } + try { + CanalInstanceExports export = exports.remove(instance.getDestination()); + if (export != null) { + export.unregister(); + } + } catch (Throwable t) { + logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t); + } + logger.info("Unregister metrics for destination {}.", instance.getDestination()); + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java new file mode 100644 index 00000000..eabc256c --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java @@ -0,0 +1,79 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import org.aspectj.lang.annotation.After; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.jctools.maps.ConcurrentAutoTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Chuanyi Li + */ +@Aspect +public class InboundThroughputAspect { + + private static final Logger logger = LoggerFactory.getLogger(InboundThroughputAspect.class); + + /** + * Support highly scalable counters + * @see ConcurrentAutoTable + */ + private static final ConcurrentAutoTable total = new ConcurrentAutoTable(); + + private static final Collector collector = new InboundThroughputCollector(); + + public static Collector getCollector() { + return collector; + } + + @Pointcut("call(byte[] com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))") + public void read() {} + + @Pointcut("call(void com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..)) ") + public void readBytes() {} + + //nested read, just eliminate them. + @Pointcut("withincode(* com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))") + public void nestedCall() {} + + @After("read() && !nestedCall() && args(len, ..)") + public void recordRead(int len) { + accumulateBytes(len); + } + + @After("readBytes() && !nestedCall() && args(.., len, timeout)") + public void recordReadBytes(int len, int timeout) { + accumulateBytes(len); + } + + private void accumulateBytes(int count) { + try { + total.add(count); + } catch (Throwable t) { + //Catch every Throwable, rather than break the business logic. + logger.warn("Error while accumulate inbound bytes.", t); + } + } + + public static class InboundThroughputCollector extends Collector { + + private InboundThroughputCollector() {} + + @Override + public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily bytes = new CounterMetricFamily("canal_net_inbound_bytes", + "Total socket inbound bytes of canal server.", + total.get()); + mfs.add(bytes); + return mfs; + } + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java new file mode 100644 index 00000000..5562b8fd --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java @@ -0,0 +1,64 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring; +import com.alibaba.otter.canal.meta.CanalMetaManager; +import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.protocol.ClientIdentity; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author Chuanyi Li + */ +public class InstanceMetaCollector extends Collector { + + private static final List InfoLabel = Arrays.asList("destination", "mode"); + + private CanalMetaManager metaManager; + + private final String destination; + + private final String mode; + + private final String subsHelp; + + public InstanceMetaCollector(CanalInstance instance) { + if (instance == null) { + throw new IllegalArgumentException("CanalInstance must not be null."); + } + if (instance instanceof CanalInstanceWithSpring) { + mode = "spring"; + } else { + mode = "manager"; + } + this.metaManager = instance.getMetaManager(); + this.destination = instance.getDestination(); + this.subsHelp = "Subscriptions of canal instance " + destination; + } + + @Override + public List collect() { + List mfs = new ArrayList(); + GaugeMetricFamily instanceInfo = new GaugeMetricFamily( + "canal_instance", + "Canal instance", + InfoLabel); + instanceInfo.addMetric(Arrays.asList(destination, mode), 1); + mfs.add(instanceInfo); + if (metaManager.isStart()) { + // client id = hardcode 1001, 目前没有意义 + List subs = metaManager.listAllSubscribeInfo(destination); + GaugeMetricFamily subscriptions = new GaugeMetricFamily( + "canal_instance_subscription", + subsHelp, CanalInstanceExports.labelList); + subscriptions.addMetric(Arrays.asList(destination), subs.size()); + mfs.add(subscriptions); + } + return mfs; + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java new file mode 100644 index 00000000..79fdca4a --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java @@ -0,0 +1,75 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.store.CanalEventStore; +import com.alibaba.otter.canal.store.CanalStoreException; +import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer; +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Chuanyi Li + */ +public class MemoryStoreCollector extends Collector { + + private static final Class clazz = MemoryEventStoreWithBuffer.class; + + private final String destination; + + private final AtomicLong putSequence; + + private final AtomicLong ackSequence; + + private final String putHelp; + + private final String ackHelp; + + public MemoryStoreCollector(CanalEventStore store, String destination) { + this.destination = destination; + if (!(store instanceof MemoryEventStoreWithBuffer)) { + throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer"); + } + MemoryEventStoreWithBuffer ms = (MemoryEventStoreWithBuffer) store; + putSequence = getDeclaredValue(ms, "putSequence"); + ackSequence = getDeclaredValue(ms, "ackSequence"); + putHelp = "Produced sequence of canal instance " + destination; + ackHelp = "Consumed sequence of canal instance " + destination; + } + + @Override + public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily put = new CounterMetricFamily("canal_instance_store_produce_seq", + putHelp, Arrays.asList(CanalInstanceExports.labels)); + put.addMetric(Collections.singletonList(destination), putSequence.doubleValue()); + mfs.add(put); + CounterMetricFamily ack = new CounterMetricFamily("canal_instance_store_consume_seq", + ackHelp, Arrays.asList(CanalInstanceExports.labels)); + ack.addMetric(Collections.singletonList(destination), ackSequence.doubleValue()); + mfs.add(ack); + return mfs; + } + + @SuppressWarnings("unchecked") + private static T getDeclaredValue(MemoryEventStoreWithBuffer store, String name) { + T value; + try { + Field putField = clazz.getDeclaredField(name); + putField.setAccessible(true); + value = (T) putField.get(store); + } catch (NoSuchFieldException e) { + throw new CanalStoreException(e); + } catch (IllegalAccessException e) { + throw new CanalStoreException(e); + } + return value; + } + +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java new file mode 100644 index 00000000..60dd808d --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java @@ -0,0 +1,80 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import io.prometheus.client.Collector; +import io.prometheus.client.CounterMetricFamily; +import org.aspectj.lang.annotation.After; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.jboss.netty.channel.Channel; +import org.jctools.maps.ConcurrentAutoTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static com.alibaba.otter.canal.server.netty.NettyUtils.HEADER_LENGTH; + +/** + * @author Chuanyi Li + */ +@Aspect +public class OutboundThroughputAspect { + private static final Logger logger = LoggerFactory.getLogger(OutboundThroughputAspect.class); + + /** + * Support highly scalable counters + * @see ConcurrentAutoTable + */ + private static final ConcurrentAutoTable total = new ConcurrentAutoTable(); + + private static final Collector collector = new OutboundThroughputCollector(); + + public static Collector getCollector() { + return collector; + } + + @Pointcut("call(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))") + public void write() {} + + //nested read, just eliminate them. + @Pointcut("withincode(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))") + public void nestedCall() {} + + @After("write() && !nestedCall() && args(ch, bytes, ..)") + public void recordWriteBytes(Channel ch, byte[] bytes) { + if (bytes != null) { + accumulateBytes(HEADER_LENGTH + bytes.length); + } + } + + @After("write() && !nestedCall() && args(ch, buf, ..)") + public void recordWriteBuffer(Channel ch, ByteBuffer buf) { + if (buf != null) { + total.add(HEADER_LENGTH + buf.limit()); + } + } + private void accumulateBytes(int count) { + try { + total.add(count); + } catch (Throwable t) { + //Catch every Throwable, rather than break the business logic. + logger.warn("Error while accumulate inbound bytes.", t); + } + } + + public static class OutboundThroughputCollector extends Collector { + + private OutboundThroughputCollector() {} + + @Override public List collect() { + List mfs = new ArrayList(); + CounterMetricFamily bytes = new CounterMetricFamily("canal_net_outbound_bytes", + "Total socket outbound bytes of canal server.", + total.get()); + mfs.add(bytes); + return mfs; + } + } +} diff --git a/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java new file mode 100644 index 00000000..c55a7eac --- /dev/null +++ b/prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java @@ -0,0 +1,73 @@ +package com.alibaba.otter.canal.prometheus.impl; + +import com.alibaba.otter.canal.prometheus.CanalInstanceExports; +import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler; +import com.alibaba.otter.canal.store.model.Event; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * @author Chuanyi Li + */ +public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler> { + + private final Collector collector; + + private long latestExecuteTime = 0L; + + private static final String DELAY_NAME = "canal_instance_traffic_delay"; + + private final String delayHelpName; + + private final List labelValues; + + public PrometheusCanalEventDownStreamHandler(final String destination) { + this.delayHelpName = "Traffic delay of canal instance " + destination + " in seconds."; + this.labelValues = Collections.singletonList(destination); + collector = new Collector() { + @Override + public List collect() { + List mfs = new ArrayList(); + long now = System.currentTimeMillis(); + GaugeMetricFamily delay = new GaugeMetricFamily( + DELAY_NAME, + delayHelpName, + CanalInstanceExports.labelList); + double d = 0.0; + if (latestExecuteTime > 0) { + d = now - latestExecuteTime; + } + d = d > 0.0 ? (d / 1000) : 0.0; + delay.addMetric(labelValues, d); + mfs.add(delay); + return mfs; + } + }; + } + + @Override + public List before(List events) { + // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming + // see: https://dev.mysql.com/worklog/task/?id=342 + // heartbeats are sent by the master only if there is no + // more unsent events in the actual binlog file for a period longer that + // master_heartbeat_period. + if (events != null && !events.isEmpty()) { + Event last = events.get(events.size() - 1); + long ts = last.getExecuteTime(); + if (ts > latestExecuteTime) { + latestExecuteTime = ts; + } + } + return events; + } + + public Collector getCollector() { + return this.collector; + } + +} diff --git a/prometheus/src/main/resources/META-INF/aop.xml b/prometheus/src/main/resources/META-INF/aop.xml new file mode 100644 index 00000000..8bd31d0c --- /dev/null +++ b/prometheus/src/main/resources/META-INF/aop.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider b/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider new file mode 100644 index 00000000..9adbf8d1 --- /dev/null +++ b/prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider @@ -0,0 +1 @@ +com.alibaba.otter.canal.prometheus.PrometheusProvider diff --git a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java index 722bb2bc..ccf21020 100644 --- a/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java +++ b/server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java @@ -1,11 +1,11 @@ package com.alibaba.otter.canal.server.embedded; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; +import com.alibaba.otter.canal.spi.CanalMetricsProvider; +import com.alibaba.otter.canal.spi.CanalMetricsService; +import com.alibaba.otter.canal.spi.NopCanalMetricsService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -33,17 +33,18 @@ import com.google.protobuf.ByteString; /** * 嵌入式版本实现 - * + * * @author jianghang 2012-7-12 下午01:34:00 * @author zebin.xuzb * @version 1.0.0 */ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, CanalService { - private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class); + private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class); private Map canalInstances; // private Map lastRollbackPostions; private CanalInstanceGenerator canalInstanceGenerator; + private CanalMetricsService metrics = NopCanalMetricsService.NOP; private static class SingletonHolder { @@ -61,7 +62,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C public void start() { if (!isStart()) { super.start(); - + // 如果存在provider,则启动metrics service + loadCanalMetrics(); + metrics.initialize(); canalInstances = MigrateMap.makeComputingMap(new Function() { public CanalInstance apply(String destination) { @@ -92,6 +95,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e); } } + metrics.terminate(); } public void start(final String destination) { @@ -99,6 +103,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C if (!canalInstance.isStart()) { try { MDC.put("destination", destination); + if (metrics.isRunning()) { + metrics.register(canalInstance); + } canalInstance.start(); logger.info("start CanalInstances[{}] successfully", destination); } finally { @@ -114,6 +121,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C try { MDC.put("destination", destination); canalInstance.stop(); + if (metrics.isRunning()) { + metrics.unregister(canalInstance); + } logger.info("stop CanalInstances[{}] successfully", destination); } finally { MDC.remove("destination"); @@ -176,7 +186,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 获取数据 - * + * *
      * 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
      * 
@@ -188,14 +198,14 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 获取数据,可以指定超时时间. - * + * *
      * 几种case:
      * a. 如果timeout为null,则采用tryGet方式,即时获取
      * b. 如果timeout不为null
      *    1. timeout为0,则采用get阻塞方式,获取数据,不设置超时,直到有足够的batchSize数据才返回
      *    2. timeout不为0,则采用get+timeout方式,获取数据,超时还没有batchSize足够的数据,有多少返回多少
-     * 
+     *
      * 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
      * 
*/ @@ -251,7 +261,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 不指定 position 获取事件。canal 会记住此 client 最新的 position。
* 如果是第一次 fetch,则会从 canal 中保存的最老一条数据开始输出。 - * + * *
      * 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
      * 
@@ -264,14 +274,14 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 不指定 position 获取事件。canal 会记住此 client 最新的 position。
* 如果是第一次 fetch,则会从 canal 中保存的最老一条数据开始输出。 - * + * *
      * 几种case:
      * a. 如果timeout为null,则采用tryGet方式,即时获取
      * b. 如果timeout不为null
      *    1. timeout为0,则采用get阻塞方式,获取数据,不设置超时,直到有足够的batchSize数据才返回
      *    2. timeout不为0,则采用get+timeout方式,获取数据,超时还没有batchSize足够的数据,有多少返回多少
-     *    
+     *
      * 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
      * 
*/ @@ -342,7 +352,7 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C /** * 进行 batch id 的确认。确认之后,小于等于此 batchId 的 Message 都会被确认。 - * + * *
      * 注意:进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
      * 
@@ -492,6 +502,27 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C } } + private void loadCanalMetrics() { + ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class); + List list = new ArrayList(); + for (CanalMetricsProvider provider : providers) { + list.add(provider); + } + if (!list.isEmpty()) { + // 发现provider, 进行初始化 + if (list.size() > 1) { + logger.warn("Found more than one CanalMetricsProvider, use the first one."); + //报告冲突 + for (CanalMetricsProvider p : list) { + logger.warn("Found CanalMetricsProvider: {}.", p.getClass().getName()); + } + } + //默认使用第一个 + CanalMetricsProvider provider = list.get(0); + this.metrics = provider.getService(); + } + } + // ========= setter ========== public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) { diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java new file mode 100644 index 00000000..ad0ceddf --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java @@ -0,0 +1,24 @@ +package com.alibaba.otter.canal.spi; + +/** + * Use java service provider mechanism to provide {@link CanalMetricsService}. + *
+ * Example:
+ * {@code
+ *     ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class);
+ *     List list = new ArrayList();
+ *     for (CanalMetricsProvider provider : providers) {
+ *         list.add(provider);
+ *     }
+ * }
+ * 
+ * @author Chuanyi Li + */ +public interface CanalMetricsProvider { + + /** + * @return Impl of {@link CanalMetricsService} + */ + CanalMetricsService getService(); + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java new file mode 100644 index 00000000..6d823be8 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java @@ -0,0 +1,42 @@ +package com.alibaba.otter.canal.spi; + +import com.alibaba.otter.canal.instance.core.CanalInstance; + +/** + * Canal server/instance metrics for export. + * + * Designed to be created by service provider. + * + * @see CanalMetricsProvider + * @author Chuanyi Li + */ +public interface CanalMetricsService { + + /** + * Initialization on canal server startup. + */ + void initialize(); + + /** + * Clean-up at canal server stop phase. + */ + void terminate(); + + /** + * @return {@code true} if the metrics service is running, otherwise {@code false}. + */ + boolean isRunning(); + + /** + * Register instance level metrics for specified instance. + * @param instance {@link CanalInstance} + */ + void register(CanalInstance instance); + + /** + * Unregister instance level metrics for specified instance. + * @param instance {@link CanalInstance} + */ + void unregister(CanalInstance instance); + +} diff --git a/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java b/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java new file mode 100644 index 00000000..0d6ad088 --- /dev/null +++ b/server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.spi; + +import com.alibaba.otter.canal.instance.core.CanalInstance; + +/** + * @author Chuanyi Li + */ +public class NopCanalMetricsService implements CanalMetricsService { + + public static final NopCanalMetricsService NOP = new NopCanalMetricsService(); + + private NopCanalMetricsService() {} + + @Override + public void initialize() { + + } + + @Override + public void terminate() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public void register(CanalInstance instance) { + + } + + @Override + public void unregister(CanalInstance instance) { + + } +}