chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #21 from alibaba/master

Browse Source 
merge
This commit is contained in:
rewerma
2018-11-14 10:48:04 +08:00
committed by GitHub
parent 8c3ade2d50 dd04577994
commit fecc112196
10 changed files with 64 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java
View File

@@ -2,13 +2,15 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLClassLoader; import java.net.URLClassLoader;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.*; import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -178,8 +180,9 @@ public class ExtensionLoader<T> {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private T createExtension(String name, String key) { private T createExtension(String name, String key) {
System.out.println("xxxxxxxxxxxxx"); // System.out.println("xxxxxxxxxxxxx");
getExtensionClasses().forEach((k, v) -> logger.info("fffff: " + k + " " + v.getName())); // getExtensionClasses().forEach((k, v) -> logger.info("fffff: " + k +
// " " + v.getName()));
Class<?> clazz = getExtensionClasses().get(name); Class<?> clazz = getExtensionClasses().get(name);
if (clazz == null) { if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type

2
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
View File

@@ -45,6 +45,8 @@ public class CanalConstants {
public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize"; public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout"; public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static String getInstanceModeKey(String destination) { public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

2
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -100,6 +100,8 @@ public class CanalLauncher {
CanalConstants.CANAL_MQ_CANALGETTIMEOUT))); CanalConstants.CANAL_MQ_CANALGETTIMEOUT)));
mqProperties.setFlatMessage(Boolean.valueOf(CanalController.getProperty(properties, mqProperties.setFlatMessage(Boolean.valueOf(CanalController.getProperty(properties,
CanalConstants.CANAL_MQ_FLATMESSAGE))); CanalConstants.CANAL_MQ_FLATMESSAGE)));
mqProperties.setCompressionType(CanalController.getProperty(properties,CanalConstants.CANAL_MQ_COMPRESSION_TYPE));
mqProperties.setAcks(CanalController.getProperty(properties,CanalConstants.CANAL_MQ_ACKS));
return mqProperties; return mqProperties;
} }

2
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
View File

@@ -304,7 +304,6 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
} }
boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable()); boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable());
if (tsdbEnable) { if (tsdbEnable) {
mysqlEventParser.setEnableTsdb(tsdbEnable);
mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() { mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() {
@Override @Override
@@ -327,6 +326,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
} }
} }
}); });
mysqlEventParser.setEnableTsdb(tsdbEnable);
} }
eventParser = mysqlEventParser; eventParser = mysqlEventParser;
} else if (type.isLocalBinlog()) { } else if (type.isLocalBinlog()) {

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
View File

@@ -57,6 +57,7 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval); ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire); ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
((DatabaseTableMeta) tableMetaTSDB).init(destination);
} }
tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -124,6 +124,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval); ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire); ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
((DatabaseTableMeta) tableMetaTSDB).init(destination);
} }
tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);

54
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -10,6 +10,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -62,9 +63,10 @@ public class DatabaseTableMeta implements TableMetaTSDB {
} }
}); });
private ReadWriteLock lock = new ReentrantReadWriteLock(); private ReadWriteLock lock = new ReentrantReadWriteLock();
private AtomicBoolean initialized = new AtomicBoolean(false);
private String destination; private String destination;
private MemoryTableMeta memoryTableMeta; private MemoryTableMeta memoryTableMeta;
private MysqlConnection connection; // 查询meta信息的链接 private volatile MysqlConnection connection; // 查询meta信息的链接
private CanalEventFilter filter; private CanalEventFilter filter;
private CanalEventFilter blackFilter; private CanalEventFilter blackFilter;
private EntryPosition lastPosition; private EntryPosition lastPosition;
@@ -74,40 +76,42 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private int snapshotInterval = 24; private int snapshotInterval = 24;
private int snapshotExpire = 360; private int snapshotExpire = 360;
private ScheduledFuture<?> scheduleSnapshotFuture; private ScheduledFuture<?> scheduleSnapshotFuture;
public DatabaseTableMeta(){ public DatabaseTableMeta(){
} }
@Override @Override
public boolean init(final String destination) { public boolean init(final String destination) {
this.destination = destination; if (initialized.compareAndSet(false, true)) {
this.memoryTableMeta = new MemoryTableMeta(); this.destination = destination;
this.memoryTableMeta = new MemoryTableMeta();
// 24小时生成一份snapshot // 24小时生成一份snapshot
if (snapshotInterval > 0) { if (snapshotInterval > 0) {
scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() { scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() {
@Override @Override
public void run() { public void run() {
boolean applyResult = false; boolean applyResult = false;
try { try {
MDC.put("destination", destination); MDC.put("destination", destination);
applyResult = applySnapshotToDB(lastPosition, false); applyResult = applySnapshotToDB(lastPosition, false);
} catch (Throwable e) { } catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e); logger.error("scheudle applySnapshotToDB faield", e);
} }
try { try {
MDC.put("destination", destination); MDC.put("destination", destination);
if (applyResult) { if (applyResult) {
snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire)); snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
}
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
} }
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
} }
} }, snapshotInterval, snapshotInterval, TimeUnit.HOURS);
}, snapshotInterval, snapshotInterval, TimeUnit.HOURS); }
} }
return true; return true;
} }

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java
View File

@@ -36,7 +36,6 @@ public class TableMetaTSDBBuilder {
} }
} }
TableMetaTSDB tableMetaTSDB = (TableMetaTSDB) applicationContext.getBean("tableMetaTSDB"); TableMetaTSDB tableMetaTSDB = (TableMetaTSDB) applicationContext.getBean("tableMetaTSDB");
tableMetaTSDB.init(destination);
logger.info("{} init TableMetaTSDB with {}", destination, springXml); logger.info("{} init TableMetaTSDB with {}", destination, springXml);
return tableMetaTSDB; return tableMetaTSDB;
} else { } else {

19
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -20,6 +20,8 @@ public class MQProperties {
private int canalBatchSize = 50; private int canalBatchSize = 50;
private Long canalGetTimeout = 100L; private Long canalGetTimeout = 100L;
private boolean flatMessage = true; private boolean flatMessage = true;
private String compressionType = "none";
private String acks = "all";
public static class CanalDestination { public static class CanalDestination {
@@ -149,4 +151,21 @@ public class MQProperties {
public void setProducerGroup(String producerGroup) { public void setProducerGroup(String producerGroup) {
this.producerGroup = producerGroup; this.producerGroup = producerGroup;
} }
public String getCompressionType() {
return compressionType;
}
public void setCompressionType(String compressionType) {
this.compressionType = compressionType;
}
public String getAcks() {
return acks;
}
public void setAcks(String acks) {
this.acks = acks;
}
} }

3
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -37,7 +37,8 @@ public class CanalKafkaProducer implements CanalMQProducer {
this.kafkaProperties = kafkaProperties; this.kafkaProperties = kafkaProperties;
Properties properties = new Properties(); Properties properties = new Properties();
properties.put("bootstrap.servers", kafkaProperties.getServers()); properties.put("bootstrap.servers", kafkaProperties.getServers());
properties.put("acks", "all"); properties.put("acks", kafkaProperties.getAcks());
properties.put("compression.type",kafkaProperties.getCompressionType());
properties.put("retries", kafkaProperties.getRetries()); properties.put("retries", kafkaProperties.getRetries());
properties.put("batch.size", kafkaProperties.getBatchSize()); properties.put("batch.size", kafkaProperties.getBatchSize());
properties.put("linger.ms", kafkaProperties.getLingerMs()); properties.put("linger.ms", kafkaProperties.getLingerMs());