chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #1047 , support snapshot expire

Browse Source
This commit is contained in:
七锋
2018-10-26 23:46:41 +08:00
parent 1e2cf9d476
commit f822761ab2
18 changed files with 176 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
View File

@@ -26,6 +26,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
protected TableMetaTSDBFactory tableMetaTSDBFactory = new DefaultTableMetaTSDBFactory();
protected boolean enableTsdb = false;
protected int tsdbSnapshotInterval = 24;
protected int tsdbSnapshotExpire = 360;
protected String tsdbSpringXml;
protected TableMetaTSDB tableMetaTSDB;
@@ -210,4 +212,20 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
return this.receivedBinlogBytes;
}
public int getTsdbSnapshotInterval() {
return tsdbSnapshotInterval;
}
public void setTsdbSnapshotInterval(int tsdbSnapshotInterval) {
this.tsdbSnapshotInterval = tsdbSnapshotInterval;
}
public int getTsdbSnapshotExpire() {
return tsdbSnapshotExpire;
}
public void setTsdbSnapshotExpire(int tsdbSnapshotExpire) {
this.tsdbSnapshotExpire = tsdbSnapshotExpire;
}
}

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
View File

@@ -55,6 +55,8 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
}
tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -121,6 +121,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
}
tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);

61
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -44,19 +44,21 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
*/
public class DatabaseTableMeta implements TableMetaTSDB {
public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
private String destination;
private MemoryTableMeta memoryTableMeta;
private MysqlConnection connection; // 查询meta信息的链接
private MysqlConnection connection; // 查询meta信息的链接
private CanalEventFilter filter;
private CanalEventFilter blackFilter;
private EntryPosition lastPosition;
private ScheduledExecutorService scheduler;
private MetaHistoryDAO metaHistoryDAO;
private MetaSnapshotDAO metaSnapshotDAO;
private int snapshotInterval = 24;
private int snapshotExpire = 360;
public DatabaseTableMeta(){
@@ -77,18 +79,27 @@ public class DatabaseTableMeta implements TableMetaTSDB {
});
// 24小时生成一份snapshot
scheduler.scheduleWithFixedDelay(new Runnable() {
if (snapshotInterval > 0) {
scheduler.scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
try {
MDC.put("destination", destination);
applySnapshotToDB(lastPosition, false);
} catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e);
@Override
public void run() {
try {
MDC.put("destination", destination);
applySnapshotToDB(lastPosition, false);
} catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e);
}
try {
MDC.put("destination", destination);
snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
}
}
}
}, 24, 24, TimeUnit.HOURS);
}, snapshotInterval, snapshotInterval, TimeUnit.HOURS);
}
return true;
}
@@ -461,6 +472,10 @@ public class DatabaseTableMeta implements TableMetaTSDB {
return true;
}
private int snapshotExpire(int expireTimestamp) {
return metaSnapshotDAO.deleteByTimestamp(destination, expireTimestamp);
}
public void setConnection(MysqlConnection connection) {
this.connection = connection;
}
@@ -489,6 +504,22 @@ public class DatabaseTableMeta implements TableMetaTSDB {
this.blackFilter = blackFilter;
}
public int getSnapshotInterval() {
return snapshotInterval;
}
public void setSnapshotInterval(int snapshotInterval) {
this.snapshotInterval = snapshotInterval;
}
public int getSnapshotExpire() {
return snapshotExpire;
}
public void setSnapshotExpire(int snapshotExpire) {
this.snapshotExpire = snapshotExpire;
}
public MysqlConnection getConnection() {
return connection;
}

11
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -37,13 +35,12 @@ public class MetaHistoryDAO extends MetaBaseDAO {
/**
* 删除interval秒之前的数据
*/
public Integer deleteByGmtModified(int interval) {
public Integer deleteByTimestamp(String destination, int interval) {
HashMap params = Maps.newHashMapWithExpectedSize(2);
long timestamp = System.currentTimeMillis() - interval * 1000;
Date date = new Date(timestamp);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
params.put("timestamp", format.format(date));
return getSqlMapClientTemplate().delete("meta_history.deleteByGmtModified", params);
params.put("timestamp", timestamp);
params.put("destination", destination);
return getSqlMapClientTemplate().delete("meta_history.deleteByTimestamp", params);
}
protected void initDao() throws Exception {

11
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java
View File

@@ -1,7 +1,5 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import com.google.common.collect.Maps;
@@ -40,13 +38,12 @@ public class MetaSnapshotDAO extends MetaBaseDAO {
/**
* 删除interval秒之前的数据
*/
public Integer deleteByGmtModified(int interval) {
public Integer deleteByTimestamp(String destination, int interval) {
HashMap params = Maps.newHashMapWithExpectedSize(2);
long timestamp = System.currentTimeMillis() - interval * 1000;
Date date = new Date(timestamp);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
params.put("timestamp", format.format(date));
return getSqlMapClientTemplate().delete("meta_snapshot.deleteByGmtModified", params);
params.put("timestamp", timestamp);
params.put("destination", destination);
return getSqlMapClientTemplate().delete("meta_snapshot.deleteByTimestamp", params);
}
protected void initDao() throws Exception {