fixed code template and format

This commit is contained in:
agapple
2015-04-28 21:11:41 +08:00
parent ea889cb004
commit 7ebb3fd1ab
86 changed files with 2763 additions and 3554 deletions

View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -27,7 +26,7 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 基于文件刷新的metaManager实现
@@ -70,36 +69,28 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>()
{
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination)
{
public File apply(String destination) {
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return loadClientIdentity(destination);
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity)
{
public Position apply(ClientIdentity clientIdentity) {
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null)
{
if (position == null) {
return nullCursor; // 返回一个空对象标识,避免出现异常
} else
{
} else {
return position;
}
}
@@ -118,10 +109,10 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
// 定时将内存中的最新值刷到file中,多次变更只刷一次
if (logger.isInfoEnabled()) {
LogPosition cursor = (LogPosition) getCursor(clientIdentity);
logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] {
clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getIdentity().getSourceAddress().toString() });
logger.info("clientId:{} cursor:[{},{},{}] address[{}]",
new Object[] { clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getIdentity().getSourceAddress().toString() });
}
flushDataToFile(clientIdentity.getDestination());
updateCursorTasks.remove(clientIdentity);
@@ -131,7 +122,10 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
}
}
}
}, period, period, TimeUnit.MILLISECONDS);
},
period,
period,
TimeUnit.MILLISECONDS);
}
public void stop() {

View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -16,6 +15,7 @@ import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.common.collect.MigrateMap;
/**
* 内存版实现
@@ -32,11 +32,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
public void start() {
super.start();
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
return MemoryClientIdentityBatch.create(clientIdentity);
}
@@ -44,11 +42,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
cursors = new MapMaker().makeMap();
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return Lists.newArrayList();
}
});
@@ -70,7 +66,7 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
if (clientIdentitys.contains(clientIdentity)) {
clientIdentitys.remove(clientIdentity);
}
clientIdentitys.add(clientIdentity);
}
@@ -167,8 +163,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
Long minBatchId = Collections.min(batches.keySet());
if (!minBatchId.equals(batchId)) {
// 检查一下提交的ack/rollback,必须按batchId分出去的顺序提交,否则容易出现丢数据
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId,
minBatchId));
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d",
batchId,
minBatchId));
}
return batches.remove(batchId);
} else {

View File

@@ -1,16 +1,18 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
/**
* 组合memory + zookeeper的使用模式
@@ -19,61 +21,48 @@ import org.springframework.util.Assert;
* @version 1.0.0
*/
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager
{
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private ExecutorService executor;
private ExecutorService executor;
private ZooKeeperMetaManager zooKeeperMetaManager;
@SuppressWarnings("serial")
private final Position nullCursor = new Position()
{
};
private final Position nullCursor = new Position() {
};
public void start()
{
public void start() {
super.start();
Assert.notNull(zooKeeperMetaManager);
if (!zooKeeperMetaManager.isStart())
{
if (!zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.start();
}
executor = Executors.newFixedThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity)
{
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null)
{
if (position == null) {
return nullCursor; // 返回一个空对象标识,避免出现异常
} else
{
} else {
return position;
}
}
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息,初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
{
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
@@ -81,12 +70,10 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
});
}
public void stop()
{
public void stop() {
super.stop();
if (zooKeeperMetaManager.isStart())
{
if (zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.stop();
}
@@ -95,73 +82,58 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
batches.clear();
}
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity);
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
}
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
super.updateCursor(clientIdentity, position);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.updateCursor(clientIdentity, position);
}
});
}
@Override
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
Position position = super.getCursor(clientIdentity);
if (position == nullCursor)
{
if (position == nullCursor) {
return null;
} else
{
} else {
return position;
}
}
public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
final Long batchId = super.addBatch(clientIdentity, positionRange);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
@@ -169,30 +141,24 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
}
public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
super.addBatch(clientIdentity, positionRange, batchId);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
}
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
}
});
@@ -200,24 +166,20 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
return positionRange;
}
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.clearAllBatchs(clientIdentity);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
}
});
}
// =============== setter / getter ================
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager)
{
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
this.zooKeeperMetaManager = zooKeeperMetaManager;
}
}

View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -20,7 +19,7 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 基于定时刷新的策略的mixed实现
@@ -53,40 +52,32 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
}
executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity)
{
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null)
{
if (position == null) {
return nullCursor; // 返回一个空对象标识,避免出现异常
} else
{
} else {
return position;
}
}
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息,初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
{
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;