chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
meta/src/main/java/com/alibaba/otter/canal/meta/CanalMetaManager.java Normal file
View File

@@ -0,0 +1,93 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
/**
* meta信息管理器
*
* @author jianghang 2012-6-14 下午09:28:48
* @author zebin.xuzb
* @version 1.0.0
*/
public interface CanalMetaManager extends CanalLifeCycle {
/**
* 增加一个 client订阅 <br/>
* 如果 client已经存在，则不做任何修改
*/
void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 判断是否订阅
*/
boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 取消client订阅
*/
void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 获取cuosr游标
*/
Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 更新cuosr游标
*/
void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException;
/**
* 根据指定的destination列出当前所有的clientIdentity信息
*/
List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException;
/**
* 获得该client最新的一个位置
*/
PositionRange getFirstBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 获得该clientId最新的一个位置
*/
PositionRange getLastestBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 为 client 产生一个唯一、递增的id
*/
Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException;
/**
* 指定batchId，插入batch数据
*/
void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
throws CanalMetaManagerException;
/**
* 根据唯一messageId，查找对应的数据起始信息
*/
PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException;
/**
* 对一个batch的确认
*/
PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException;
/**
* 查询当前的所有batch信息
*/
Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException;
/**
* 清除对应的batch信息
*/
void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException;
}

375
meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java Normal file
View File

@@ -0,0 +1,375 @@
package com.alibaba.otter.canal.meta;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
/**
* 基于文件刷新的metaManager实现
*
* <pre>
* 策略：
* 1. 先写内存，然后定时刷新数据到File
* 2. 数据采取overwrite模式(只保留最后一次)，通过logger实施append模式(记录历史版本)
* </pre>
*
* @author jianghang 2013-4-15 下午05:55:57
* @version 1.0.4
*/
public class FileMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private static final Logger logger = LoggerFactory.getLogger(FileMixedMetaManager.class);
private static final Charset charset = Charset.forName("UTF-8");
private File dataDir;
private String dataFileName = "meta.dat";
private Map<String, File> dataFileCaches;
private ScheduledExecutorService executor;
@SuppressWarnings("serial")
private final Position nullCursor = new Position() {
};
private long period = 1000; // 单位ms
private Set<ClientIdentity> updateCursorTasks;
public void start() {
super.start();
Assert.notNull(dataDir);
if (!dataDir.exists()) {
try {
FileUtils.forceMkdir(dataDir);
} catch (IOException e) {
throw new CanalMetaManagerException(e);
}
}
if (!dataDir.canRead() || !dataDir.canWrite()) {
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = new MapMaker().makeComputingMap(new Function<String, File>() {
public File apply(String destination) {
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination) {
return loadClientIdentity(destination);
}
});
cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity) {
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
}
});
updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
// 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() {
public void run() {
List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
for (ClientIdentity clientIdentity : tasks) {
MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
try {
// 定时将内存中的最新值刷到file中，多次变更只刷一次
if (logger.isInfoEnabled()) {
LogPosition cursor = (LogPosition) getCursor(clientIdentity);
logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] {
clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getIdentity().getSourceAddress().toString() });
}
flushDataToFile(clientIdentity.getDestination());
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
}
}
}
}, period, period, TimeUnit.MILLISECONDS);
}
public void stop() {
super.stop();
flushDataToFile();// 刷新数据
executor.shutdownNow();
destinations.clear();
batches.clear();
}
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
flushDataToFile(clientIdentity.getDestination());
}
});
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
flushDataToFile(clientIdentity.getDestination());
}
});
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
updateCursorTasks.add(clientIdentity);// 添加到任务队列中进行触发
super.updateCursor(clientIdentity, position);
}
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
Position position = super.getCursor(clientIdentity);
if (position == nullCursor) {
return null;
} else {
return position;
}
}
// ============================ helper method ======================
private File getDataFile(String destination) {
File destinationMetaDir = new File(dataDir, destination);
if (!destinationMetaDir.exists()) {
try {
FileUtils.forceMkdir(destinationMetaDir);
} catch (IOException e) {
throw new CanalMetaManagerException(e);
}
}
return new File(destinationMetaDir, dataFileName);
}
private FileMetaInstanceData loadDataFromFile(File dataFile) {
try {
if (!dataFile.exists()) {
return null;
}
String json = FileUtils.readFileToString(dataFile, charset.name());
return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
} catch (IOException e) {
throw new CanalMetaManagerException(e);
}
}
private void flushDataToFile() {
for (String destination : destinations.keySet()) {
flushDataToFile(destination);
}
}
private void flushDataToFile(String destination) {
flushDataToFile(destination, dataFileCaches.get(destination));
}
private void flushDataToFile(String destination, File dataFile) {
FileMetaInstanceData data = new FileMetaInstanceData();
if (destinations.containsKey(destination)) {
synchronized (destination.intern()) { // 基于destination控制一下并发更新
data.setDestination(destination);
List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
List<ClientIdentity> clientIdentitys = destinations.get(destination);
for (ClientIdentity clientIdentity : clientIdentitys) {
FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
clientData.setClientIdentity(clientIdentity);
Position position = cursors.get(clientIdentity);
if (position != null && position != nullCursor) {
clientData.setCursor((LogPosition) position);
}
clientDatas.add(clientData);
}
data.setClientDatas(clientDatas);
}
String json = JsonUtils.marshalToString(data);
try {
FileUtils.writeStringToFile(dataFile, json);
} catch (IOException e) {
throw new CanalMetaManagerException(e);
}
}
}
private List<ClientIdentity> loadClientIdentity(String destination) {
List<ClientIdentity> result = Lists.newArrayList();
FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
if (data == null) {
return result;
}
List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
if (clientDatas == null) {
return result;
}
for (FileMetaClientIdentityData clientData : clientDatas) {
if (clientData.getClientIdentity().getDestination().equals(destination)) {
result.add(clientData.getClientIdentity());
}
}
return result;
}
private Position loadCursor(String destination, ClientIdentity clientIdentity) {
FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
if (data == null) {
return null;
}
List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
if (clientDatas == null) {
return null;
}
for (FileMetaClientIdentityData clientData : clientDatas) {
if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
return clientData.getCursor();
}
}
return null;
}
/**
* 描述一个clientIdentity对应的数据对象
*
* @author jianghang 2013-4-15 下午06:19:40
* @version 1.0.4
*/
public static class FileMetaClientIdentityData {
private ClientIdentity clientIdentity;
private LogPosition cursor;
public FileMetaClientIdentityData(){
}
public FileMetaClientIdentityData(ClientIdentity clientIdentity, MemoryClientIdentityBatch batch,
LogPosition cursor){
this.clientIdentity = clientIdentity;
this.cursor = cursor;
}
public ClientIdentity getClientIdentity() {
return clientIdentity;
}
public void setClientIdentity(ClientIdentity clientIdentity) {
this.clientIdentity = clientIdentity;
}
public Position getCursor() {
return cursor;
}
public void setCursor(LogPosition cursor) {
this.cursor = cursor;
}
}
/**
* 描述整个canal instance对应数据对象
*
* @author jianghang 2013-4-15 下午06:20:22
* @version 1.0.4
*/
public static class FileMetaInstanceData {
private String destination;
private List<FileMetaClientIdentityData> clientDatas;
public FileMetaInstanceData(){
}
public FileMetaInstanceData(String destination, List<FileMetaClientIdentityData> clientDatas){
this.destination = destination;
this.clientDatas = clientDatas;
}
public String getDestination() {
return destination;
}
public void setDestination(String destination) {
this.destination = destination;
}
public List<FileMetaClientIdentityData> getClientDatas() {
return clientDatas;
}
public void setClientDatas(List<FileMetaClientIdentityData> clientDatas) {
this.clientDatas = clientDatas;
}
}
public void setDataDir(String dataDir) {
this.dataDir = new File(dataDir);
}
public void setDataDir(File dataDir) {
this.dataDir = dataDir;
}
public void setPeriod(long period) {
this.period = period;
}
}

226
meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java Normal file
View File

@@ -0,0 +1,226 @@
package com.alibaba.otter.canal.meta;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
/**
* 内存版实现
*
* @author zebin.xuzb @ 2012-7-2
* @version 1.0.0
*/
public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
protected Map<String, List<ClientIdentity>> destinations;
protected Map<ClientIdentity, MemoryClientIdentityBatch> batches;
protected Map<ClientIdentity, Position> cursors;
public void start() {
super.start();
batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
return MemoryClientIdentityBatch.create(clientIdentity);
}
});
cursors = new MapMaker().makeMap();
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination) {
return Lists.newArrayList();
}
});
}
public void stop() {
super.stop();
destinations.clear();
cursors.clear();
for (MemoryClientIdentityBatch batch : batches.values()) {
batch.clearPositionRanges();
}
}
public synchronized void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
List<ClientIdentity> clientIdentitys = destinations.get(clientIdentity.getDestination());
if (clientIdentitys.contains(clientIdentity)) {
clientIdentitys.remove(clientIdentity);
}
clientIdentitys.add(clientIdentity);
}
public synchronized boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
List<ClientIdentity> clientIdentitys = destinations.get(clientIdentity.getDestination());
return clientIdentitys != null && clientIdentitys.contains(clientIdentity);
}
public synchronized void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
List<ClientIdentity> clientIdentitys = destinations.get(clientIdentity.getDestination());
if (clientIdentitys != null && clientIdentitys.contains(clientIdentity)) {
clientIdentitys.remove(clientIdentity);
}
}
public synchronized List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
return destinations.get(destination);
}
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
return cursors.get(clientIdentity);
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
cursors.put(clientIdentity, position);
}
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
return batches.get(clientIdentity).addPositionRange(positionRange);
}
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
throws CanalMetaManagerException {
batches.get(clientIdentity).addPositionRange(positionRange, batchId);// 添加记录到指定batchId
}
public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
return batches.get(clientIdentity).removePositionRange(batchId);
}
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
return batches.get(clientIdentity).getPositionRange(batchId);
}
public PositionRange getLastestBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException {
return batches.get(clientIdentity).getLastestPositionRange();
}
public PositionRange getFirstBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException {
return batches.get(clientIdentity).getFirstPositionRange();
}
public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
return batches.get(clientIdentity).listAllPositionRange();
}
public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
batches.get(clientIdentity).clearPositionRanges();
}
// ============================
public static class MemoryClientIdentityBatch {
private ClientIdentity clientIdentity;
private Map<Long, PositionRange> batches = new MapMaker().makeMap();
private AtomicLong atomicMaxBatchId = new AtomicLong(1);
public static MemoryClientIdentityBatch create(ClientIdentity clientIdentity) {
return new MemoryClientIdentityBatch(clientIdentity);
}
public MemoryClientIdentityBatch(){
}
protected MemoryClientIdentityBatch(ClientIdentity clientIdentity){
this.clientIdentity = clientIdentity;
}
public synchronized void addPositionRange(PositionRange positionRange, Long batchId) {
updateMaxId(batchId);
batches.put(batchId, positionRange);
}
public synchronized Long addPositionRange(PositionRange positionRange) {
Long batchId = atomicMaxBatchId.getAndIncrement();
batches.put(batchId, positionRange);
return batchId;
}
public synchronized PositionRange removePositionRange(Long batchId) {
if (batches.containsKey(batchId)) {
Long minBatchId = Collections.min(batches.keySet());
if (!minBatchId.equals(batchId)) {
// 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId,
minBatchId));
}
return batches.remove(batchId);
} else {
return null;
}
}
public synchronized PositionRange getPositionRange(Long batchId) {
return batches.get(batchId);
}
public synchronized PositionRange getLastestPositionRange() {
if (batches.size() == 0) {
return null;
} else {
Long batchId = Collections.max(batches.keySet());
return batches.get(batchId);
}
}
public synchronized PositionRange getFirstPositionRange() {
if (batches.size() == 0) {
return null;
} else {
Long batchId = Collections.min(batches.keySet());
return batches.get(batchId);
}
}
public synchronized Map<Long, PositionRange> listAllPositionRange() {
Set<Long> batchIdSets = batches.keySet();
List<Long> batchIds = Lists.newArrayList(batchIdSets);
Collections.sort(Lists.newArrayList(batchIds));
return Maps.newHashMap(batches);
}
public synchronized void clearPositionRanges() {
batches.clear();
}
private synchronized void updateMaxId(Long batchId) {
if (atomicMaxBatchId.get() < batchId + 1) {
atomicMaxBatchId.set(batchId + 1);
}
}
// ============ setter & getter =========
public ClientIdentity getClientIdentity() {
return clientIdentity;
}
public void setClientIdentity(ClientIdentity clientIdentity) {
this.clientIdentity = clientIdentity;
}
}
}

185
meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java Normal file
View File

@@ -0,0 +1,185 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
/**
* 组合memory + zookeeper的使用模式
*
* @author jianghang 2012-7-11 下午03:58:00
* @version 1.0.0
*/
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private ExecutorService executor;
private ZooKeeperMetaManager zooKeeperMetaManager;
@SuppressWarnings("serial")
private final Position nullCursor = new Position() {
};
public void start() {
super.start();
Assert.notNull(zooKeeperMetaManager);
if (!zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.start();
}
executor = Executors.newFixedThreadPool(1);
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
}
});
batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
}
});
}
public void stop() {
super.stop();
if (zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.stop();
}
executor.shutdownNow();
destinations.clear();
batches.clear();
}
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity);
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
}
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
throws CanalMetaManagerException {
super.updateCursor(clientIdentity, position);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.updateCursor(clientIdentity, position);
}
});
}
@Override
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
Position position = super.getCursor(clientIdentity);
if (position == nullCursor) {
return null;
} else {
return position;
}
}
public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
throws CanalMetaManagerException {
final Long batchId = super.addBatch(clientIdentity, positionRange);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
return batchId;
}
public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
throws CanalMetaManagerException {
super.addBatch(clientIdentity, positionRange, batchId);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
}
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
throws CanalMetaManagerException {
PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
}
});
return positionRange;
}
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.clearAllBatchs(clientIdentity);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
}
});
}
// =============== setter / getter ================
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
this.zooKeeperMetaManager = zooKeeperMetaManager;
}
}

168
meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java Normal file
View File

@@ -0,0 +1,168 @@
package com.alibaba.otter.canal.meta;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
/**
* 基于定时刷新的策略的mixed实现
*
* <pre>
* 几个优化：
* 1. 去除batch数据刷新到zk中，切换时batch数据可忽略，重新从头开始获取
* 2. cursor的更新，启用定时刷新，合并多次请求。如果最近没有变化则不更新
* </pre>
*
* @author jianghang 2012-9-11 下午02:41:15
* @version 1.0.0
*/
public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private static final Logger logger = LoggerFactory.getLogger(PeriodMixedMetaManager.class);
private ScheduledExecutorService executor;
private ZooKeeperMetaManager zooKeeperMetaManager;
@SuppressWarnings("serial")
private final Position nullCursor = new Position() {
};
private long period = 1000; // 单位ms
private Set<ClientIdentity> updateCursorTasks;
public void start() {
super.start();
Assert.notNull(zooKeeperMetaManager);
if (!zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.start();
}
executor = Executors.newScheduledThreadPool(1);
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
}
});
batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
}
});
updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
// 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() {
public void run() {
List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
for (ClientIdentity clientIdentity : tasks) {
try {
// 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
}
}
}
}, period, period, TimeUnit.MILLISECONDS);
}
public void stop() {
super.stop();
if (zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.stop();
}
executor.shutdownNow();
destinations.clear();
batches.clear();
}
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
updateCursorTasks.add(clientIdentity);// 添加到任务队列中进行触发
super.updateCursor(clientIdentity, position);
}
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
Position position = super.getCursor(clientIdentity);
if (position == nullCursor) {
return null;
} else {
return position;
}
}
// =============== setter / getter ================
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
this.zooKeeperMetaManager = zooKeeperMetaManager;
}
public void setPeriod(long period) {
this.period = period;
}
}

329
meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java Normal file
View File

@@ -0,0 +1,329 @@
package com.alibaba.otter.canal.meta;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
import org.apache.commons.lang.StringUtils;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
* zk 版本的 canal manager， 存储结构：
*
* <pre>
* /otter
* canal
* destinations
* dest1
* client1
* filter
* batch_mark
* 1
* 2
* 3
* </pre>
*
* @author zebin.xuzb @ 2012-6-21
* @author jianghang
* @version 1.0.0
*/
public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
private static final String ENCODE = "UTF-8";
private ZkClientx zkClientx;
public void start() {
super.start();
Assert.notNull(zkClientx);
}
public void stop() {
super.stop();
}
public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
clientIdentity.getClientId());
try {
zkClientx.createPersistent(path, true);
} catch (ZkNodeExistsException e) {
// ignore
}
if (clientIdentity.hasFilter()) {
String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
byte[] bytes = null;
try {
bytes = clientIdentity.getFilter().getBytes(ENCODE);
} catch (UnsupportedEncodingException e) {
throw new CanalMetaManagerException(e);
}
try {
zkClientx.createPersistent(filterPath, bytes);
} catch (ZkNodeExistsException e) {
// ignore
zkClientx.writeData(filterPath, bytes);
}
}
}
public boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
clientIdentity.getClientId());
return zkClientx.exists(path);
}
public void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
clientIdentity.getClientId());
zkClientx.deleteRecursive(path); // 递归删除所有信息
}
public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getDestinationPath(destination);
List<String> childs = null;
try {
childs = zkClientx.getChildren(path);
} catch (ZkNoNodeException e) {
// ignore
}
if (CollectionUtils.isEmpty(childs)) {
return new ArrayList<ClientIdentity>();
}
List<Short> clientIds = new ArrayList<Short>();
for (String child : childs) {
if (StringUtils.isNumeric(child)) {
clientIds.add(ZookeeperPathUtils.getClientId(child));
}
}
Collections.sort(clientIds); // 进行一个排序
List<ClientIdentity> clientIdentities = Lists.newArrayList();
for (Short clientId : clientIds) {
path = ZookeeperPathUtils.getFilterPath(destination, clientId);
byte[] bytes = zkClientx.readData(path, true);
String filter = null;
if (bytes != null) {
try {
filter = new String(bytes, ENCODE);
} catch (UnsupportedEncodingException e) {
throw new CanalMetaManagerException(e);
}
}
clientIdentities.add(new ClientIdentity(destination, clientId, filter));
}
return clientIdentities;
}
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
byte[] data = zkClientx.readData(path, true);
if (data == null || data.length == 0) {
return null;
}
return JsonUtils.unmarshalFromByte(data, Position.class);
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
try {
zkClientx.writeData(path, data);
} catch (ZkNoNodeException e) {
zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
}
}
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
String batchPath = zkClientx.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR,
data,
true);
String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(),
batchId);
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
zkClientx.createPersistent(path, data, true);
}
public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = zkClientx.getChildren(batchsPath);
if (CollectionUtils.isEmpty(nodes)) {
// 没有batch记录
return null;
}
// 找到最小的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
Long minBatchId = Collections.min(batchIds);
if (!minBatchId.equals(batchId)) {
// 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
}
if (!batchIds.contains(batchId)) {
// 不存在对应的batchId
return null;
}
PositionRange positionRange = getBatch(clientIdentity, batchId);
if (positionRange != null) {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(),
batchId);
zkClientx.delete(path);
}
return positionRange;
}
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(),
batchId);
byte[] data = zkClientx.readData(path, true);
if (data == null) {
return null;
}
PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
return positionRange;
}
public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
List<String> batchChilds = zkClientx.getChildren(path);
for (String batchChild : batchChilds) {
String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
zkClientx.delete(batchPath);
}
}
public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
List<String> nodes = null;
try {
nodes = zkClientx.getChildren(path);
} catch (ZkNoNodeException e) {
// ignore
}
if (CollectionUtils.isEmpty(nodes)) {
return null;
}
// 找到最大的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
Long maxBatchId = Collections.max(batchIds);
PositionRange result = getBatch(clientIdentity, maxBatchId);
if (result == null) { // 出现为null，说明zk节点有变化，重新获取
return getLastestBatch(clientIdentity);
} else {
return result;
}
}
public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
List<String> nodes = null;
try {
nodes = zkClientx.getChildren(path);
} catch (ZkNoNodeException e) {
// ignore
}
if (CollectionUtils.isEmpty(nodes)) {
return null;
}
// 找到最小的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
Long minBatchId = Collections.min(batchIds);
PositionRange result = getBatch(clientIdentity, minBatchId);
if (result == null) { // 出现为null，说明zk节点有变化，重新获取
return getFirstBatch(clientIdentity);
} else {
return result;
}
}
public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
List<String> nodes = null;
try {
nodes = zkClientx.getChildren(path);
} catch (ZkNoNodeException e) {
// ignore
}
if (CollectionUtils.isEmpty(nodes)) {
return Maps.newHashMap();
}
// 找到最大的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
Collections.sort(batchIds); // 从小到大排序
Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap();
for (Long batchId : batchIds) {
PositionRange result = getBatch(clientIdentity, batchId);
if (result == null) {// 出现为null，说明zk节点有变化，重新获取
return listAllBatchs(clientIdentity);
} else {
positionRanges.put(batchId, result);
}
}
return positionRanges;
}
// =========== setter ==========
public void setZkClientx(ZkClientx zkClientx) {
this.zkClientx = zkClientx;
}
}

33
meta/src/main/java/com/alibaba/otter/canal/meta/exception/CanalMetaManagerException.java Normal file
View File

@@ -0,0 +1,33 @@
package com.alibaba.otter.canal.meta.exception;
import com.alibaba.otter.canal.common.CanalException;
/**
* @author zebin.xuzb @ 2012-6-21
* @version 1.0.0
*/
public class CanalMetaManagerException extends CanalException {
private static final long serialVersionUID = -654893533794556357L;
public CanalMetaManagerException(String errorCode){
super(errorCode);
}
public CanalMetaManagerException(String errorCode, Throwable cause){
super(errorCode, cause);
}
public CanalMetaManagerException(String errorCode, String errorDesc){
super(errorCode + ":" + errorDesc);
}
public CanalMetaManagerException(String errorCode, String errorDesc, Throwable cause){
super(errorCode + ":" + errorDesc, cause);
}
public CanalMetaManagerException(Throwable cause){
super(cause);
}
}

117
meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java Normal file
View File

@@ -0,0 +1,117 @@
package com.alibaba.otter.canal.meta;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import junit.framework.Assert;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.protocol.position.LogIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class AbstractMetaManagerTest extends AbstractZkTest {
private static final String MYSQL_ADDRESS = "127.0.0.1";
protected ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1); ;
public void doSubscribeTest(CanalMetaManager metaManager) {
ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
metaManager.subscribe(client1);
metaManager.subscribe(client1); // 重复调用
ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
metaManager.subscribe(client2);
List<ClientIdentity> clients = metaManager.listAllSubscribeInfo(destination);
Assert.assertEquals(Arrays.asList(client1, client2), clients);
metaManager.unsubscribe(client2);
ClientIdentity client3 = new ClientIdentity(destination, (short) 3);
metaManager.subscribe(client3);
clients = metaManager.listAllSubscribeInfo(destination);
Assert.assertEquals(Arrays.asList(client1, client3), clients);
}
public void doBatchTest(CanalMetaManager metaManager) {
metaManager.subscribe(clientIdentity);
PositionRange first = metaManager.getFirstBatch(clientIdentity);
PositionRange lastest = metaManager.getLastestBatch(clientIdentity);
Assert.assertNull(first);
Assert.assertNull(lastest);
PositionRange range1 = buildRange(1);
Long batchId1 = metaManager.addBatch(clientIdentity, range1);
PositionRange range2 = buildRange(2);
Long batchId2 = metaManager.addBatch(clientIdentity, range2);
Assert.assertEquals((batchId1.longValue() + 1), batchId2.longValue());
// 验证get
PositionRange getRange1 = metaManager.getBatch(clientIdentity, batchId1);
Assert.assertEquals(range1, getRange1);
PositionRange getRange2 = metaManager.getBatch(clientIdentity, batchId2);
Assert.assertEquals(range2, getRange2);
PositionRange range3 = buildRange(3);
Long batchId3 = batchId2 + 1;
metaManager.addBatch(clientIdentity, range3, batchId3);
PositionRange range4 = buildRange(4);
Long batchId4 = metaManager.addBatch(clientIdentity, range4);
Assert.assertEquals((batchId3.longValue() + 1), batchId4.longValue());
// 验证remove
metaManager.removeBatch(clientIdentity, batchId1);
range1 = metaManager.getBatch(clientIdentity, batchId1);
Assert.assertNull(range1);
// 验证first / lastest
first = metaManager.getFirstBatch(clientIdentity);
lastest = metaManager.getLastestBatch(clientIdentity);
Assert.assertEquals(range2, first);
Assert.assertEquals(range4, lastest);
Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
Assert.assertEquals(3, ranges.size());
}
public Position doCursorTest(CanalMetaManager metaManager) {
metaManager.subscribe(clientIdentity);
Position position1 = metaManager.getCursor(clientIdentity);
Assert.assertNull(position1);
PositionRange range = buildRange(1);
metaManager.updateCursor(clientIdentity, range.getStart());
Position position2 = metaManager.getCursor(clientIdentity);
Assert.assertEquals(range.getStart(), position2);
metaManager.updateCursor(clientIdentity, range.getEnd());
Position position3 = metaManager.getCursor(clientIdentity);
Assert.assertEquals(range.getEnd(), position3);
return position3;
}
private PositionRange<LogPosition> buildRange(int number) {
LogPosition start = new LogPosition();
start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
LogPosition end = new LogPosition();
end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
return new PositionRange<LogPosition>(start, end);
}
}

18
meta/src/test/java/com/alibaba/otter/canal/meta/AbstractZkTest.java Normal file
View File

@@ -0,0 +1,18 @@
package com.alibaba.otter.canal.meta;
import org.junit.Assert;
public class AbstractZkTest {
protected String destination = "ljhtest1";
protected String cluster1 = "127.0.0.1:2188";
protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188";
public void sleep(long time) {
try {
Thread.sleep(time);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
}
}

88
meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java Normal file
View File

@@ -0,0 +1,88 @@
package com.alibaba.otter.canal.meta;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class FileMixedMetaManagerTest extends AbstractMetaManagerTest {
private static final String tmp = System.getProperty("java.io.tmpdir", "/tmp");
private static final File dataDir = new File(tmp, "canal");
@Before
public void setUp() {
try {
FileUtils.deleteDirectory(dataDir);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
@Test
public void testSubscribeAll() {
FileMixedMetaManager metaManager = new FileMixedMetaManager();
metaManager.setDataDir(dataDir);
metaManager.setPeriod(100);
metaManager.start();
doSubscribeTest(metaManager);
sleep(2000L);
// 重新构建一次，能获得上一次zk上的记录
FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
metaManager2.setDataDir(dataDir);
metaManager2.setPeriod(100);
metaManager2.start();
List<ClientIdentity> clients = metaManager2.listAllSubscribeInfo(destination);
Assert.assertEquals(2, clients.size());
metaManager.stop();
}
@Test
public void testBatchAll() {
FileMixedMetaManager metaManager = new FileMixedMetaManager();
metaManager.setDataDir(dataDir);
metaManager.setPeriod(100);
metaManager.start();
doBatchTest(metaManager);
metaManager.clearAllBatchs(clientIdentity);
Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
Assert.assertEquals(0, ranges.size());
metaManager.stop();
}
@Test
public void testCursorAll() {
FileMixedMetaManager metaManager = new FileMixedMetaManager();
metaManager.setDataDir(dataDir);
metaManager.setPeriod(100);
metaManager.start();
Position lastPosition = doCursorTest(metaManager);
sleep(1000L);
// 重新构建一次，能获得上一次zk上的记录
FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
metaManager2.setDataDir(dataDir);
metaManager2.setPeriod(100);
metaManager2.start();
Position position = metaManager2.getCursor(clientIdentity);
Assert.assertEquals(position, lastPosition);
metaManager.stop();
}
}

40
meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java Normal file
View File

@@ -0,0 +1,40 @@
package com.alibaba.otter.canal.meta;
import java.util.Map;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class MemoryMetaManagerTest extends AbstractMetaManagerTest {
@Test
public void testSubscribeAll() {
MemoryMetaManager metaManager = new MemoryMetaManager();
metaManager.start();
doSubscribeTest(metaManager);
metaManager.stop();
}
@Test
public void testBatchAll() {
MemoryMetaManager metaManager = new MemoryMetaManager();
metaManager.start();
doBatchTest(metaManager);
metaManager.clearAllBatchs(clientIdentity);
Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
Assert.assertEquals(0, ranges.size());
metaManager.stop();
}
@Test
public void testCursorAll() {
MemoryMetaManager metaManager = new MemoryMetaManager();
metaManager.start();
doCursorTest(metaManager);
metaManager.stop();
}
}

104
meta/src/test/java/com/alibaba/otter/canal/meta/MixedMetaManagerTest.java Normal file
View File

@@ -0,0 +1,104 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class MixedMetaManagerTest extends AbstractMetaManagerTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testSubscribeAll() {
MixedMetaManager metaManager = new MixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
doSubscribeTest(metaManager);
sleep(1000L);
// 重新构建一次，能获得上一次zk上的记录
MixedMetaManager metaManager2 = new MixedMetaManager();
metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager2.start();
List<ClientIdentity> clients = metaManager2.listAllSubscribeInfo(destination);
Assert.assertEquals(2, clients.size());
metaManager.stop();
}
@Test
public void testBatchAll() {
MixedMetaManager metaManager = new MixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
doBatchTest(metaManager);
sleep(1000L);
// 重新构建一次，能获得上一次zk上的记录
MixedMetaManager metaManager2 = new MixedMetaManager();
metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager2.start();
Map<Long, PositionRange> ranges = metaManager2.listAllBatchs(clientIdentity);
Assert.assertEquals(3, ranges.size());
metaManager.clearAllBatchs(clientIdentity);
ranges = metaManager.listAllBatchs(clientIdentity);
Assert.assertEquals(0, ranges.size());
metaManager.stop();
metaManager2.stop();
}
@Test
public void testCursorAll() {
MixedMetaManager metaManager = new MixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
Position lastPosition = doCursorTest(metaManager);
sleep(1000L);
// 重新构建一次，能获得上一次zk上的记录
MixedMetaManager metaManager2 = new MixedMetaManager();
metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager2.start();
Position position = metaManager2.getCursor(clientIdentity);
Assert.assertEquals(position, lastPosition);
metaManager.stop();
}
}

94
meta/src/test/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManagerTest.java Normal file
View File

@@ -0,0 +1,94 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class PeriodMixedMetaManagerTest extends AbstractMetaManagerTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testSubscribeAll() {
PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
doSubscribeTest(metaManager);
sleep(1000L);
// 重新构建一次，能获得上一次zk上的记录
PeriodMixedMetaManager metaManager2 = new PeriodMixedMetaManager();
metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager2.start();
List<ClientIdentity> clients = metaManager2.listAllSubscribeInfo(destination);
Assert.assertEquals(2, clients.size());
metaManager.stop();
}
@Test
public void testBatchAll() {
PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
doBatchTest(metaManager);
metaManager.clearAllBatchs(clientIdentity);
Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
Assert.assertEquals(0, ranges.size());
metaManager.stop();
}
@Test
public void testCursorAll() {
PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(zkclientx);
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
Position lastPosition = doCursorTest(metaManager);
sleep(1000L);
// 重新构建一次，能获得上一次zk上的记录
PeriodMixedMetaManager metaManager2 = new PeriodMixedMetaManager();
metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager2.start();
Position position = metaManager2.getCursor(clientIdentity);
Assert.assertEquals(position, lastPosition);
metaManager.stop();
}
}

62
meta/src/test/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManagerTest.java Normal file
View File

@@ -0,0 +1,62 @@
package com.alibaba.otter.canal.meta;
import java.util.Map;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.protocol.position.PositionRange;
public class ZooKeeperMetaManagerTest extends AbstractMetaManagerTest {
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
@Before
public void setUp() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@After
public void tearDown() {
String path = ZookeeperPathUtils.getDestinationPath(destination);
zkclientx.deleteRecursive(path);
}
@Test
public void testSubscribeAll() {
ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
metaManager.setZkClientx(zkclientx);
metaManager.start();
doSubscribeTest(metaManager);
metaManager.stop();
}
@Test
public void testBatchAll() {
ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
metaManager.setZkClientx(zkclientx);
metaManager.start();
doBatchTest(metaManager);
metaManager.clearAllBatchs(clientIdentity);
Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
Assert.assertEquals(0, ranges.size());
metaManager.stop();
}
@Test
public void testCursorhAll() {
ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
metaManager.setZkClientx(zkclientx);
metaManager.start();
doCursorTest(metaManager);
metaManager.stop();
}
}