chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve metrics.

Browse Source
This commit is contained in:
Chuanyi Li
2018-08-19 16:51:08 +08:00
parent 2656d1d2af
commit 2784f33b52
15 changed files with 335 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java
View File

@@ -7,7 +7,6 @@ import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
import com.google.common.base.Preconditions;
import io.prometheus.client.Collector;
import io.prometheus.client.Counter;
import io.prometheus.client.CounterMetricFamily;
import io.prometheus.client.GaugeMetricFamily;
import org.slf4j.Logger;
@@ -30,12 +29,8 @@ public class EntryCollector extends Collector implements InstanceRegistry {
private static final Logger logger = LoggerFactory.getLogger(SinkCollector.class);
private static final String DELAY = "canal_instance_traffic_delay";
private static final String TRANSACTION = "canal_instance_transactions";
private static final String ROW_EVENTS = "canal_instance_row_events";
private static final String ROWS_COUNTER = "canal_instance_rows_counter";
private static final String DELAY_HELP = "Traffic delay of canal instance in milliseconds";
private static final String TRANSACTION_HELP = "Transactions counter of canal instance";
private static final String ROW_EVENTS_HELP = "Rowdata events counter of canal instance";
private static final String ROWS_COUNTER_HELP = "Rows counter of canal instance";
private final ConcurrentMap<String, EntryMetricsHolder> instances = new ConcurrentHashMap<String, EntryMetricsHolder>();
private EntryCollector() {}
@@ -55,10 +50,6 @@ public class EntryCollector extends Collector implements InstanceRegistry {
DELAY_HELP, DEST_LABELS_LIST);
CounterMetricFamily transactions = new CounterMetricFamily(TRANSACTION,
TRANSACTION_HELP, DEST_LABELS_LIST);
CounterMetricFamily rowEvents = new CounterMetricFamily(ROW_EVENTS,
ROW_EVENTS_HELP, DEST_LABELS_LIST);
CounterMetricFamily rowsCounter = new CounterMetricFamily(ROWS_COUNTER,
ROWS_COUNTER_HELP, DEST_LABELS_LIST);
for (EntryMetricsHolder emh : instances.values()) {
long now = System.currentTimeMillis();
long latest = emh.latestExecTime.get();
@@ -66,13 +57,9 @@ public class EntryCollector extends Collector implements InstanceRegistry {
delay.addMetric(emh.destLabelValues, (now - latest));
}
transactions.addMetric(emh.destLabelValues, emh.transactionCounter.doubleValue());
rowEvents.addMetric(emh.destLabelValues, emh.rowEventCounter.doubleValue());
rowsCounter.addMetric(emh.destLabelValues, emh.rowsCounter.doubleValue());
}
mfs.add(delay);
mfs.add(transactions);
mfs.add(rowEvents);
mfs.add(rowsCounter);
return mfs;
}
@@ -89,12 +76,8 @@ public class EntryCollector extends Collector implements InstanceRegistry {
PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink);
holder.latestExecTime = handler.getLatestExecuteTime();
holder.transactionCounter = handler.getTransactionCounter();
holder.rowEventCounter = handler.getRowEventCounter();
holder.rowsCounter = handler.getRowsCounter();
Preconditions.checkNotNull(holder.latestExecTime);
Preconditions.checkNotNull(holder.transactionCounter);
Preconditions.checkNotNull(holder.rowEventCounter);
Preconditions.checkNotNull(holder.rowsCounter);
EntryMetricsHolder old = instances.put(destination, holder);
if (old != null) {
logger.warn("Remove stale EntryCollector for instance {}.", destination);
@@ -145,8 +128,6 @@ public class EntryCollector extends Collector implements InstanceRegistry {
private class EntryMetricsHolder {
private AtomicLong latestExecTime;
private AtomicLong transactionCounter;
private AtomicLong rowEventCounter;
private AtomicLong rowsCounter;
private List<String> destLabelValues;
}

11
prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java
View File

@@ -15,8 +15,6 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow
private final AtomicLong latestExecuteTime = new AtomicLong(System.currentTimeMillis());
private final AtomicLong transactionCounter = new AtomicLong(0L);
private final AtomicLong rowEventCounter = new AtomicLong(0L);
private final AtomicLong rowsCounter = new AtomicLong(0L);
@Override
public List<Event> before(List<Event> events) {
@@ -34,8 +32,6 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow
case ROWDATA: {
long exec = e.getExecuteTime();
if (exec > 0) localExecTime = exec;
rowEventCounter.incrementAndGet();
rowsCounter.addAndGet(e.getRowsCount());
break;
}
case TRANSACTIONEND: {
@@ -80,11 +76,4 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow
return transactionCounter;
}
public AtomicLong getRowsCounter() {
return rowsCounter;
}
public AtomicLong getRowEventCounter() {
return rowEventCounter;
}
}

5
prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java
View File

@@ -71,7 +71,6 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler
String destination = result.getDestination();
PacketType type = result.getType();
outboundCounter.labels(destination).inc(result.getAmount());
packetsCounter.labels(destination, type.name()).inc();
short errorCode = result.getErrorCode();
if (errorCode > 0) {
errorsCounter.labels(destination, Short.toString(errorCode)).inc();
@@ -81,12 +80,16 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler
switch (type) {
case GET:
boolean empty = result.getEmpty();
// 区分一下空包
if (empty) {
emptyBatchesCounter.labels(destination).inc();
} else {
packetsCounter.labels(destination, type.name()).inc();
}
break;
// reserve for others
default:
packetsCounter.labels(destination, type.name()).inc();
break;
}
}

59
prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java
View File

@@ -34,11 +34,23 @@ public class StoreCollector extends Collector implements InstanceRegistry {
private static final String STORE = "canal_instance_store";
private static final String PRODUCE_MEM = "canal_instance_store_produce_mem";
private static final String CONSUME_MEM = "canal_instance_store_consume_mem";
private static final String PUT_DELAY = "canal_instance_put_delay";
private static final String GET_DELAY = "canal_instance_get_delay";
private static final String ACK_DELAY = "canal_instance_ack_delay";
private static final String PUT_ROWS = "canal_instance_put_rows";
private static final String GET_ROWS = "canal_instance_get_rows";
private static final String ACK_ROWS = "canal_instance_ack_rows";
private static final String PRODUCE_HELP = "Produced events counter of canal instance";
private static final String CONSUME_HELP = "Consumed events counter of canal instance";
private static final String STORE_HELP = "Canal instance info";
private static final String PRODUCE_MEM_HELP = "Produced mem bytes of canal instance";
private static final String CONSUME_MEM_HELP = "Consumed mem bytes of canal instance";
private static final String PUT_DELAY_HELP = "Traffic delay of canal instance put";
private static final String GET_DELAY_HELP = "Traffic delay of canal instance get";
private static final String ACK_DELAY_HELP = "Traffic delay of canal instance ack";
private static final String PUT_ROWS_HELP = "Put table rows of canal instance";
private static final String GET_ROWS_HELP = "Got table rows of canal instance";
private static final String ACK_ROWS_HELP = "Acked table rows of canal instance";
private final ConcurrentMap<String, StoreMetricsHolder> instances = new ConcurrentHashMap<String, StoreMetricsHolder>();
private final List<String> storeLabelsList = Arrays.asList(DEST, "batchMode", "size");
@@ -65,11 +77,40 @@ public class StoreCollector extends Collector implements InstanceRegistry {
PRODUCE_MEM_HELP, DEST_LABELS_LIST);
CounterMetricFamily ackMem = new CounterMetricFamily(CONSUME_MEM,
CONSUME_MEM_HELP, DEST_LABELS_LIST);
GaugeMetricFamily putDelay = new GaugeMetricFamily(PUT_DELAY,
PUT_DELAY_HELP, DEST_LABELS_LIST);
GaugeMetricFamily getDelay = new GaugeMetricFamily(GET_DELAY,
GET_DELAY_HELP, DEST_LABELS_LIST);
GaugeMetricFamily ackDelay = new GaugeMetricFamily(ACK_DELAY,
ACK_DELAY_HELP, DEST_LABELS_LIST);
CounterMetricFamily putRows = new CounterMetricFamily(PUT_ROWS,
PUT_ROWS_HELP, DEST_LABELS_LIST);
CounterMetricFamily getRows = new CounterMetricFamily(GET_ROWS,
GET_ROWS_HELP, DEST_LABELS_LIST);
CounterMetricFamily ackRows = new CounterMetricFamily(ACK_ROWS,
ACK_ROWS_HELP, DEST_LABELS_LIST);
boolean hasMem = false;
for (StoreMetricsHolder smh : instances.values()) {
final boolean isMem = smh.batchMode.isMemSize();
put.addMetric(smh.destLabelValues, smh.putSeq.doubleValue());
ack.addMetric(smh.destLabelValues, smh.ackSeq.doubleValue());
long pet = smh.putExecTime.get();
// 防止出现启动时，未消费造成的get, ack延时小于前阶段的情况
long get = Math.min(smh.getExecTime.get(), pet);
long aet = Math.min(smh.ackExecTime.get(), get);
long now = System.currentTimeMillis();
if (now >= pet) {
putDelay.addMetric(smh.destLabelValues, (now - pet));
}
if (now >= get) {
getDelay.addMetric(smh.destLabelValues, (now - get));
}
if (now >= aet) {
ackDelay.addMetric(smh.destLabelValues, (now - aet));
}
putRows.addMetric(smh.destLabelValues, smh.putTableRows.doubleValue());
getRows.addMetric(smh.destLabelValues, smh.getTableRows.doubleValue());
ackRows.addMetric(smh.destLabelValues, smh.ackTableRows.doubleValue());
store.addMetric(smh.storeLabelValues, 1);
if (isMem) {
hasMem = true;
@@ -80,6 +121,12 @@ public class StoreCollector extends Collector implements InstanceRegistry {
mfs.add(put);
mfs.add(ack);
mfs.add(store);
mfs.add(putDelay);
mfs.add(getDelay);
mfs.add(ackDelay);
mfs.add(putRows);
mfs.add(getRows);
mfs.add(ackRows);
if (hasMem) {
mfs.add(putMem);
mfs.add(ackMem);
@@ -102,6 +149,12 @@ public class StoreCollector extends Collector implements InstanceRegistry {
holder.destLabelValues = Collections.singletonList(destination);
holder.size = memStore.getBufferSize();
holder.storeLabelValues = Arrays.asList(destination, holder.batchMode.name(), Integer.toString(holder.size));
holder.putExecTime = memStore.getPutExecTime();
holder.getExecTime = memStore.getGetExecTime();
holder.ackExecTime = memStore.getAckExecTime();
holder.putTableRows = memStore.getPutTableRows();
holder.getTableRows = memStore.getGetTableRows();
holder.ackTableRows = memStore.getAckTableRows();
Preconditions.checkNotNull(holder.batchMode);
Preconditions.checkNotNull(holder.putSeq);
Preconditions.checkNotNull(holder.ackSeq);
@@ -129,6 +182,12 @@ public class StoreCollector extends Collector implements InstanceRegistry {
private BatchMode batchMode;
private AtomicLong putMemSize;
private AtomicLong ackMemSize;
private AtomicLong putExecTime;
private AtomicLong getExecTime;
private AtomicLong ackExecTime;
private AtomicLong putTableRows;
private AtomicLong getTableRows;
private AtomicLong ackTableRows;
private int size;
private List<String> destLabelValues;
private List<String> storeLabelValues;