mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed code template and format
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
package com.alibaba.otter.canal.meta;
|
||||
|
||||
import com.google.common.collect.MigrateMap;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
@@ -27,7 +26,7 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
|
||||
import com.alibaba.otter.canal.protocol.position.Position;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
|
||||
/**
|
||||
* 基于文件刷新的metaManager实现
|
||||
@@ -70,36 +69,28 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
|
||||
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
|
||||
}
|
||||
|
||||
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>()
|
||||
{
|
||||
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
|
||||
|
||||
public File apply(String destination)
|
||||
{
|
||||
public File apply(String destination) {
|
||||
return getDataFile(destination);
|
||||
}
|
||||
});
|
||||
|
||||
executor = Executors.newScheduledThreadPool(1);
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
|
||||
{
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
|
||||
|
||||
public List<ClientIdentity> apply(String destination)
|
||||
{
|
||||
public List<ClientIdentity> apply(String destination) {
|
||||
return loadClientIdentity(destination);
|
||||
}
|
||||
});
|
||||
|
||||
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
|
||||
{
|
||||
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
|
||||
|
||||
public Position apply(ClientIdentity clientIdentity)
|
||||
{
|
||||
public Position apply(ClientIdentity clientIdentity) {
|
||||
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
|
||||
if (position == null)
|
||||
{
|
||||
if (position == null) {
|
||||
return nullCursor; // 返回一个空对象标识,避免出现异常
|
||||
} else
|
||||
{
|
||||
} else {
|
||||
return position;
|
||||
}
|
||||
}
|
||||
@@ -118,10 +109,10 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
|
||||
// 定时将内存中的最新值刷到file中,多次变更只刷一次
|
||||
if (logger.isInfoEnabled()) {
|
||||
LogPosition cursor = (LogPosition) getCursor(clientIdentity);
|
||||
logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] {
|
||||
clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
|
||||
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
|
||||
cursor.getIdentity().getSourceAddress().toString() });
|
||||
logger.info("clientId:{} cursor:[{},{},{}] address[{}]",
|
||||
new Object[] { clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
|
||||
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
|
||||
cursor.getIdentity().getSourceAddress().toString() });
|
||||
}
|
||||
flushDataToFile(clientIdentity.getDestination());
|
||||
updateCursorTasks.remove(clientIdentity);
|
||||
@@ -131,7 +122,10 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
|
||||
}
|
||||
}
|
||||
}
|
||||
}, period, period, TimeUnit.MILLISECONDS);
|
||||
},
|
||||
period,
|
||||
period,
|
||||
TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.alibaba.otter.canal.meta;
|
||||
|
||||
import com.google.common.collect.MigrateMap;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -16,6 +15,7 @@ import com.google.common.base.Function;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
|
||||
/**
|
||||
* 内存版实现
|
||||
@@ -32,11 +32,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
|
||||
public void start() {
|
||||
super.start();
|
||||
|
||||
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
|
||||
{
|
||||
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
|
||||
|
||||
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
|
||||
{
|
||||
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
|
||||
return MemoryClientIdentityBatch.create(clientIdentity);
|
||||
}
|
||||
|
||||
@@ -44,11 +42,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
|
||||
|
||||
cursors = new MapMaker().makeMap();
|
||||
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
|
||||
{
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
|
||||
|
||||
public List<ClientIdentity> apply(String destination)
|
||||
{
|
||||
public List<ClientIdentity> apply(String destination) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
});
|
||||
@@ -70,7 +66,7 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
|
||||
if (clientIdentitys.contains(clientIdentity)) {
|
||||
clientIdentitys.remove(clientIdentity);
|
||||
}
|
||||
|
||||
|
||||
clientIdentitys.add(clientIdentity);
|
||||
}
|
||||
|
||||
@@ -167,8 +163,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
|
||||
Long minBatchId = Collections.min(batches.keySet());
|
||||
if (!minBatchId.equals(batchId)) {
|
||||
// 检查一下提交的ack/rollback,必须按batchId分出去的顺序提交,否则容易出现丢数据
|
||||
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId,
|
||||
minBatchId));
|
||||
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d",
|
||||
batchId,
|
||||
minBatchId));
|
||||
}
|
||||
return batches.remove(batchId);
|
||||
} else {
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
package com.alibaba.otter.canal.meta;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
|
||||
import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||
import com.alibaba.otter.canal.protocol.position.Position;
|
||||
import com.alibaba.otter.canal.protocol.position.PositionRange;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* 组合memory + zookeeper的使用模式
|
||||
@@ -19,61 +21,48 @@ import org.springframework.util.Assert;
|
||||
* @version 1.0.0
|
||||
*/
|
||||
|
||||
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager
|
||||
{
|
||||
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
|
||||
|
||||
private ExecutorService executor;
|
||||
private ExecutorService executor;
|
||||
private ZooKeeperMetaManager zooKeeperMetaManager;
|
||||
@SuppressWarnings("serial")
|
||||
private final Position nullCursor = new Position()
|
||||
{
|
||||
};
|
||||
private final Position nullCursor = new Position() {
|
||||
};
|
||||
|
||||
public void start()
|
||||
{
|
||||
public void start() {
|
||||
super.start();
|
||||
Assert.notNull(zooKeeperMetaManager);
|
||||
if (!zooKeeperMetaManager.isStart())
|
||||
{
|
||||
if (!zooKeeperMetaManager.isStart()) {
|
||||
zooKeeperMetaManager.start();
|
||||
}
|
||||
|
||||
executor = Executors.newFixedThreadPool(1);
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
|
||||
{
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
|
||||
|
||||
public List<ClientIdentity> apply(String destination)
|
||||
{
|
||||
public List<ClientIdentity> apply(String destination) {
|
||||
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
|
||||
}
|
||||
});
|
||||
|
||||
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
|
||||
{
|
||||
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
|
||||
|
||||
public Position apply(ClientIdentity clientIdentity)
|
||||
{
|
||||
public Position apply(ClientIdentity clientIdentity) {
|
||||
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
|
||||
if (position == null)
|
||||
{
|
||||
if (position == null) {
|
||||
return nullCursor; // 返回一个空对象标识,避免出现异常
|
||||
} else
|
||||
{
|
||||
} else {
|
||||
return position;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
|
||||
{
|
||||
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
|
||||
|
||||
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
|
||||
{
|
||||
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
|
||||
// 读取一下zookeeper信息,初始化一次
|
||||
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
|
||||
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
|
||||
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
|
||||
{
|
||||
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
|
||||
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
|
||||
}
|
||||
return batches;
|
||||
@@ -81,12 +70,10 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
|
||||
});
|
||||
}
|
||||
|
||||
public void stop()
|
||||
{
|
||||
public void stop() {
|
||||
super.stop();
|
||||
|
||||
if (zooKeeperMetaManager.isStart())
|
||||
{
|
||||
if (zooKeeperMetaManager.isStart()) {
|
||||
zooKeeperMetaManager.stop();
|
||||
}
|
||||
|
||||
@@ -95,73 +82,58 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
|
||||
batches.clear();
|
||||
}
|
||||
|
||||
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
|
||||
{
|
||||
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
|
||||
super.subscribe(clientIdentity);
|
||||
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.subscribe(clientIdentity);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
|
||||
{
|
||||
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
|
||||
super.unsubscribe(clientIdentity);
|
||||
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.unsubscribe(clientIdentity);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
|
||||
throws CanalMetaManagerException
|
||||
{
|
||||
throws CanalMetaManagerException {
|
||||
super.updateCursor(clientIdentity, position);
|
||||
|
||||
// 异步刷新
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.updateCursor(clientIdentity, position);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException
|
||||
{
|
||||
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
|
||||
Position position = super.getCursor(clientIdentity);
|
||||
if (position == nullCursor)
|
||||
{
|
||||
if (position == nullCursor) {
|
||||
return null;
|
||||
} else
|
||||
{
|
||||
} else {
|
||||
return position;
|
||||
}
|
||||
}
|
||||
|
||||
public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
|
||||
throws CanalMetaManagerException
|
||||
{
|
||||
throws CanalMetaManagerException {
|
||||
final Long batchId = super.addBatch(clientIdentity, positionRange);
|
||||
// 异步刷新
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
|
||||
}
|
||||
});
|
||||
@@ -169,30 +141,24 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
|
||||
}
|
||||
|
||||
public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
|
||||
throws CanalMetaManagerException
|
||||
{
|
||||
throws CanalMetaManagerException {
|
||||
super.addBatch(clientIdentity, positionRange, batchId);
|
||||
// 异步刷新
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
|
||||
throws CanalMetaManagerException
|
||||
{
|
||||
throws CanalMetaManagerException {
|
||||
PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
|
||||
// 异步刷新
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
|
||||
}
|
||||
});
|
||||
@@ -200,24 +166,20 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
|
||||
return positionRange;
|
||||
}
|
||||
|
||||
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException
|
||||
{
|
||||
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
|
||||
super.clearAllBatchs(clientIdentity);
|
||||
|
||||
// 异步刷新
|
||||
executor.submit(new Runnable()
|
||||
{
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run()
|
||||
{
|
||||
public void run() {
|
||||
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// =============== setter / getter ================
|
||||
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager)
|
||||
{
|
||||
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
|
||||
this.zooKeeperMetaManager = zooKeeperMetaManager;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.alibaba.otter.canal.meta;
|
||||
|
||||
import com.google.common.collect.MigrateMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
@@ -20,7 +19,7 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||
import com.alibaba.otter.canal.protocol.position.Position;
|
||||
import com.alibaba.otter.canal.protocol.position.PositionRange;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
|
||||
/**
|
||||
* 基于定时刷新的策略的mixed实现
|
||||
@@ -53,40 +52,32 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
|
||||
}
|
||||
|
||||
executor = Executors.newScheduledThreadPool(1);
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
|
||||
{
|
||||
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
|
||||
|
||||
public List<ClientIdentity> apply(String destination)
|
||||
{
|
||||
public List<ClientIdentity> apply(String destination) {
|
||||
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
|
||||
}
|
||||
});
|
||||
|
||||
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
|
||||
{
|
||||
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
|
||||
|
||||
public Position apply(ClientIdentity clientIdentity)
|
||||
{
|
||||
public Position apply(ClientIdentity clientIdentity) {
|
||||
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
|
||||
if (position == null)
|
||||
{
|
||||
if (position == null) {
|
||||
return nullCursor; // 返回一个空对象标识,避免出现异常
|
||||
} else
|
||||
{
|
||||
} else {
|
||||
return position;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
|
||||
{
|
||||
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
|
||||
{
|
||||
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
|
||||
|
||||
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
|
||||
// 读取一下zookeeper信息,初始化一次
|
||||
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
|
||||
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
|
||||
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
|
||||
{
|
||||
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
|
||||
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
|
||||
}
|
||||
return batches;
|
||||
|
||||
Reference in New Issue
Block a user