Canal monitoring(prometheus module) snapshot version.

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalInstanceExports.java

@@ -0,0 +1,98 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.prometheus.impl.InstanceMetaCollector;
+import com.alibaba.otter.canal.prometheus.impl.MemoryStoreCollector;
+import com.alibaba.otter.canal.prometheus.impl.PrometheusCanalEventDownStreamHandler;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.entry.EntryEventSink;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+public class CanalInstanceExports {
+
+    private static final Logger      logger         = LoggerFactory.getLogger(CanalInstanceExports.class);
+
+    public static final String[]     labels         = {"destination"};
+
+    public static final List<String> labelList      = Collections.singletonList(labels[0]);
+
+    private final String             destination;
+
+    private Collector                storeCollector;
+
+    private Collector                delayCollector;
+
+    private Collector                metaCollector;
+
+    private CanalInstanceExports(CanalInstance instance) {
+        this.destination = instance.getDestination();
+        initDelayGauge(instance);
+        initStoreCollector(instance);
+        initMetaCollector(instance);
+    }
+
+
+
+    static CanalInstanceExports forInstance(CanalInstance instance) {
+        return new CanalInstanceExports(instance);
+    }
+
+    void register() {
+        if (delayCollector != null) {
+            delayCollector.register();
+        }
+        if (storeCollector != null) {
+            storeCollector.register();
+        }
+        if (metaCollector != null) {
+            metaCollector.register();
+        }
+    }
+
+    void unregister() {
+        if (delayCollector != null) {
+            CollectorRegistry.defaultRegistry.unregister(delayCollector);
+        }
+        if (storeCollector != null) {
+            CollectorRegistry.defaultRegistry.unregister(storeCollector);
+        }
+        if (metaCollector != null) {
+            CollectorRegistry.defaultRegistry.unregister(metaCollector);
+        }
+    }
+
+    private void initDelayGauge(CanalInstance instance) {
+        CanalEventSink sink = instance.getEventSink();
+        if (sink instanceof EntryEventSink) {
+            EntryEventSink entryEventSink = (EntryEventSink) sink;
+            // TODO ensure not to add handler again
+            PrometheusCanalEventDownStreamHandler handler = new PrometheusCanalEventDownStreamHandler(destination);
+            entryEventSink.addHandler(handler);
+            delayCollector = handler.getCollector();
+        } else {
+            logger.warn("This impl register metrics for only EntryEventSink, skip.");
+        }
+    }
+
+    private void initStoreCollector(CanalInstance instance) {
+        try {
+            storeCollector = new MemoryStoreCollector(instance.getEventStore(), destination);
+        } catch (CanalStoreException cse) {
+            logger.warn("Failed to register metrics for destination {}.", destination, cse);
+        }
+    }
+
+    private void initMetaCollector(CanalInstance instance) {
+        metaCollector = new InstanceMetaCollector(instance);
+    }
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/CanalServerExports.java

@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.prometheus.impl.InboundThroughputAspect;
+import com.alibaba.otter.canal.prometheus.impl.OutboundThroughputAspect;
+
+/**
+ * @author Chuanyi Li
+ */
+public class CanalServerExports {
+
+    private static boolean initialized = false;
+
+    public static synchronized void initialize() {
+        if (!initialized) {
+            InboundThroughputAspect.getCollector().register();
+            OutboundThroughputAspect.getCollector().register();
+            initialized = true;
+        }
+    }
+
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java

@@ -0,0 +1,15 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.spi.CanalMetricsProvider;
+import com.alibaba.otter.canal.spi.CanalMetricsService;
+
+/**
+ * @author Chuanyi Li
+ */
+public class PrometheusProvider implements CanalMetricsProvider {
+
+    @Override
+    public CanalMetricsService getService() {
+        return PrometheusService.getInstance();
+    }
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java

@@ -0,0 +1,110 @@
+package com.alibaba.otter.canal.prometheus;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.spi.CanalMetricsService;
+import io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.client.hotspot.DefaultExports;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author Chuanyi Li
+ */
+public class PrometheusService implements CanalMetricsService {
+
+    private static final Logger                           logger  = LoggerFactory.getLogger(PrometheusService.class);
+
+    private final Map<String, CanalInstanceExports>       exports = new ConcurrentHashMap<String, CanalInstanceExports>();
+
+    private volatile boolean                              running = false;
+
+    private HTTPServer                                    server;
+
+    private PrometheusService() {
+    }
+
+    private static class SingletonHolder {
+        private static final PrometheusService SINGLETON = new PrometheusService();
+    }
+
+    public static PrometheusService getInstance() {
+        return SingletonHolder.SINGLETON;
+    }
+
+    @Override
+    public void initialize() {
+        try {
+            //TODO 1.Configurable port
+            //TODO 2.Https
+            server = new HTTPServer(11112);
+        } catch (IOException e) {
+            logger.warn("Unable to start prometheus HTTPServer.", e);
+            return;
+        }
+        try {
+            // JVM exports
+            DefaultExports.initialize();
+            // Canal server level exports
+            CanalServerExports.initialize();
+        } catch (Throwable t) {
+            logger.warn("Unable to initialize server exports.", t);
+        }
+
+        running = true;
+    }
+
+    @Override
+    public void terminate() {
+        running = false;
+        // Normally, service should be terminated at canal shutdown.
+        // No need to unregister instance exports explicitly.
+        // But for the sake of safety, unregister them.
+        for (CanalInstanceExports ie : exports.values()) {
+            ie.unregister();
+        }
+        if (server != null) {
+            server.stop();
+        }
+    }
+
+    @Override
+    public boolean isRunning() {
+        return running;
+    }
+
+    @Override
+    public void register(CanalInstance instance) {
+        if (instance.isStart()) {
+            logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination());
+            return;
+        }
+        try {
+            CanalInstanceExports export = CanalInstanceExports.forInstance(instance);
+            export.register();
+            exports.put(instance.getDestination(), export);
+        } catch (Throwable t) {
+            logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t);
+        }
+        logger.info("Register metrics for destination {}.", instance.getDestination());
+    }
+
+    @Override
+    public void unregister(CanalInstance instance) {
+        if (instance.isStart()) {
+            logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination());
+        }
+        try {
+            CanalInstanceExports export = exports.remove(instance.getDestination());
+            if (export != null) {
+                export.unregister();
+            }
+        } catch (Throwable t) {
+            logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t);
+        }
+        logger.info("Unregister metrics for destination {}.", instance.getDestination());
+    }
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InboundThroughputAspect.java

@@ -0,0 +1,79 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.jctools.maps.ConcurrentAutoTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+@Aspect
+public class InboundThroughputAspect {
+
+    private static final Logger              logger    = LoggerFactory.getLogger(InboundThroughputAspect.class);
+
+    /**
+     *  Support highly scalable counters
+     *  @see ConcurrentAutoTable
+     */
+    private static final ConcurrentAutoTable total     = new ConcurrentAutoTable();
+
+    private static final Collector           collector = new InboundThroughputCollector();
+
+    public static Collector getCollector() {
+        return collector;
+    }
+
+    @Pointcut("call(byte[] com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))")
+    public void read() {}
+
+    @Pointcut("call(void com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..)) ")
+    public void readBytes() {}
+
+    //nested read, just eliminate them.
+    @Pointcut("withincode(* com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel.read(..))")
+    public void nestedCall() {}
+
+    @After("read() && !nestedCall()  && args(len, ..)")
+    public void recordRead(int len) {
+        accumulateBytes(len);
+    }
+
+    @After("readBytes() && !nestedCall() && args(.., len, timeout)")
+    public void recordReadBytes(int len, int timeout) {
+        accumulateBytes(len);
+    }
+
+    private void accumulateBytes(int count) {
+        try {
+            total.add(count);
+        } catch (Throwable t) {
+            //Catch every Throwable, rather than break the business logic.
+            logger.warn("Error while accumulate inbound bytes.", t);
+        }
+    }
+
+    public static class InboundThroughputCollector extends Collector {
+
+        private InboundThroughputCollector() {}
+
+        @Override
+        public List<MetricFamilySamples> collect() {
+            List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
+            CounterMetricFamily bytes = new CounterMetricFamily("canal_net_inbound_bytes",
+                    "Total socket inbound bytes of canal server.",
+                    total.get());
+            mfs.add(bytes);
+            return mfs;
+        }
+    }
+
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/InstanceMetaCollector.java

@@ -0,0 +1,64 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.prometheus.CanalInstanceExports;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import io.prometheus.client.Collector;
+import io.prometheus.client.GaugeMetricFamily;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+public class InstanceMetaCollector extends Collector {
+
+    private static final List<String> InfoLabel    = Arrays.asList("destination", "mode");
+
+    private CanalMetaManager          metaManager;
+
+    private final String              destination;
+
+    private final String              mode;
+
+    private final String              subsHelp;
+
+    public InstanceMetaCollector(CanalInstance instance) {
+        if (instance == null) {
+            throw new IllegalArgumentException("CanalInstance must not be null.");
+        }
+        if (instance instanceof CanalInstanceWithSpring) {
+            mode = "spring";
+        } else {
+            mode = "manager";
+        }
+        this.metaManager = instance.getMetaManager();
+        this.destination = instance.getDestination();
+        this.subsHelp = "Subscriptions of canal instance " + destination;
+    }
+
+    @Override
+    public List<MetricFamilySamples> collect() {
+        List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
+        GaugeMetricFamily instanceInfo = new GaugeMetricFamily(
+                "canal_instance",
+                "Canal instance",
+                InfoLabel);
+        instanceInfo.addMetric(Arrays.asList(destination, mode), 1);
+        mfs.add(instanceInfo);
+        if (metaManager.isStart()) {
+            // client id = hardcode 1001, 目前没有意义
+            List<ClientIdentity> subs = metaManager.listAllSubscribeInfo(destination);
+            GaugeMetricFamily subscriptions = new GaugeMetricFamily(
+                    "canal_instance_subscription",
+                    subsHelp, CanalInstanceExports.labelList);
+            subscriptions.addMetric(Arrays.asList(destination), subs.size());
+            mfs.add(subscriptions);
+        }
+        return mfs;
+    }
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/MemoryStoreCollector.java

@@ -0,0 +1,75 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import com.alibaba.otter.canal.prometheus.CanalInstanceExports;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * @author Chuanyi Li
+ */
+public class MemoryStoreCollector extends Collector {
+
+    private static final Class<MemoryEventStoreWithBuffer> clazz  = MemoryEventStoreWithBuffer.class;
+
+    private final String                                   destination;
+
+    private final AtomicLong                               putSequence;
+
+    private final AtomicLong                               ackSequence;
+
+    private final String                                   putHelp;
+
+    private final String                                   ackHelp;
+
+    public MemoryStoreCollector(CanalEventStore store, String destination) {
+        this.destination = destination;
+        if (!(store instanceof MemoryEventStoreWithBuffer)) {
+            throw new IllegalArgumentException("EventStore must be MemoryEventStoreWithBuffer");
+        }
+        MemoryEventStoreWithBuffer ms = (MemoryEventStoreWithBuffer) store;
+        putSequence = getDeclaredValue(ms, "putSequence");
+        ackSequence = getDeclaredValue(ms, "ackSequence");
+        putHelp = "Produced sequence of canal instance " + destination;
+        ackHelp = "Consumed sequence of canal instance " + destination;
+    }
+
+    @Override
+    public List<MetricFamilySamples> collect() {
+        List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
+        CounterMetricFamily put = new CounterMetricFamily("canal_instance_store_produce_seq",
+                putHelp, Arrays.asList(CanalInstanceExports.labels));
+        put.addMetric(Collections.singletonList(destination), putSequence.doubleValue());
+        mfs.add(put);
+        CounterMetricFamily ack = new CounterMetricFamily("canal_instance_store_consume_seq",
+                ackHelp, Arrays.asList(CanalInstanceExports.labels));
+        ack.addMetric(Collections.singletonList(destination), ackSequence.doubleValue());
+        mfs.add(ack);
+        return mfs;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T getDeclaredValue(MemoryEventStoreWithBuffer store, String name) {
+        T value;
+        try {
+            Field putField = clazz.getDeclaredField(name);
+            putField.setAccessible(true);
+            value = (T) putField.get(store);
+        } catch (NoSuchFieldException e) {
+            throw new CanalStoreException(e);
+        } catch (IllegalAccessException e) {
+            throw new CanalStoreException(e);
+        }
+        return value;
+    }
+
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/OutboundThroughputAspect.java

@@ -0,0 +1,80 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.jboss.netty.channel.Channel;
+import org.jctools.maps.ConcurrentAutoTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.alibaba.otter.canal.server.netty.NettyUtils.HEADER_LENGTH;
+
+/**
+ * @author Chuanyi Li
+ */
+@Aspect
+public class OutboundThroughputAspect {
+    private static final Logger              logger    = LoggerFactory.getLogger(OutboundThroughputAspect.class);
+
+    /**
+     *  Support highly scalable counters
+     *  @see ConcurrentAutoTable
+     */
+    private static final ConcurrentAutoTable total     = new ConcurrentAutoTable();
+
+    private static final Collector           collector = new OutboundThroughputCollector();
+
+    public static Collector getCollector() {
+        return collector;
+    }
+
+    @Pointcut("call(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))")
+    public void write() {}
+
+    //nested read, just eliminate them.
+    @Pointcut("withincode(* com.alibaba.otter.canal.server.netty.NettyUtils.write(..))")
+    public void nestedCall() {}
+
+    @After("write() && !nestedCall() && args(ch, bytes, ..)")
+    public void recordWriteBytes(Channel ch, byte[] bytes) {
+        if (bytes != null) {
+            accumulateBytes(HEADER_LENGTH + bytes.length);
+        }
+    }
+
+    @After("write() && !nestedCall() && args(ch, buf, ..)")
+    public void recordWriteBuffer(Channel ch, ByteBuffer buf) {
+        if (buf != null) {
+            total.add(HEADER_LENGTH + buf.limit());
+        }
+    }
+    private void accumulateBytes(int count) {
+        try {
+            total.add(count);
+        } catch (Throwable t) {
+            //Catch every Throwable, rather than break the business logic.
+            logger.warn("Error while accumulate inbound bytes.", t);
+        }
+    }
+
+    public static class OutboundThroughputCollector extends Collector {
+
+        private OutboundThroughputCollector() {}
+
+        @Override public List<MetricFamilySamples> collect() {
+            List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
+            CounterMetricFamily bytes = new CounterMetricFamily("canal_net_outbound_bytes",
+                    "Total socket outbound bytes of canal server.",
+                    total.get());
+            mfs.add(bytes);
+            return mfs;
+        }
+    }
+}

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java

@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.prometheus.impl;
+
+import com.alibaba.otter.canal.prometheus.CanalInstanceExports;
+import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler;
+import com.alibaba.otter.canal.store.model.Event;
+import io.prometheus.client.Collector;
+import io.prometheus.client.GaugeMetricFamily;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author Chuanyi Li
+ */
+public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler<List<Event>> {
+
+    private final Collector     collector;
+
+    private long                latestExecuteTime = 0L;
+
+    private static final String DELAY_NAME        = "canal_instance_traffic_delay";
+
+    private final String        delayHelpName;
+
+    private final List<String>  labelValues;
+
+    public PrometheusCanalEventDownStreamHandler(final String destination) {
+        this.delayHelpName = "Traffic delay of canal instance " + destination + " in seconds.";
+        this.labelValues = Collections.singletonList(destination);
+        collector = new Collector() {
+            @Override
+            public List<MetricFamilySamples> collect() {
+                List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
+                long now = System.currentTimeMillis();
+                GaugeMetricFamily delay = new GaugeMetricFamily(
+                        DELAY_NAME,
+                        delayHelpName,
+                        CanalInstanceExports.labelList);
+                double d = 0.0;
+                if (latestExecuteTime > 0) {
+                    d = now - latestExecuteTime;
+                }
+                d = d > 0.0 ? (d / 1000) : 0.0;
+                delay.addMetric(labelValues, d);
+                mfs.add(delay);
+                return mfs;
+            }
+        };
+    }
+
+    @Override
+    public List<Event> before(List<Event> events) {
+        // TODO utilize MySQL master heartbeat packet to refresh delay if always no more events coming
+        // see: https://dev.mysql.com/worklog/task/?id=342
+        // heartbeats are sent by the master only if there is no
+        // more unsent events in the actual binlog file for a period longer that
+        // master_heartbeat_period.
+        if (events != null && !events.isEmpty()) {
+            Event last = events.get(events.size() - 1);
+            long ts = last.getExecuteTime();
+            if (ts > latestExecuteTime) {
+                latestExecuteTime = ts;
+            }
+        }
+        return events;
+    }
+
+    public Collector getCollector() {
+        return this.collector;
+    }
+
+}

prometheus/src/main/resources/META-INF/aop.xml

@@ -0,0 +1,11 @@
+<aspectj>
+
+    <aspects>
+        <aspect name="com.alibaba.otter.canal.prometheus.impl.InboundThroughputAspect"/>
+        <aspect name="com.alibaba.otter.canal.prometheus.impl.OutboundThroughputAspect"/>
+    </aspects>
+    <weaver options="-verbose -showWeaveInfo">
+        <include within="com.alibaba.otter.canal..*"/>
+    </weaver>
+
+</aspectj>

prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider

@@ -0,0 +1 @@
+com.alibaba.otter.canal.prometheus.PrometheusProvider