chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
rewerma
2019-01-17 10:48:14 +08:00
committed by GitHub
parent d2bbe9b061 171bedf488
commit 4beb1e6855
4 changed files with 458 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -88,44 +88,46 @@ public class RdbSyncService {
* @param function 回调方法 * @param function 回调方法
*/ */
public void sync(List<Dml> dmls, Function<Dml, Boolean> function) { public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
boolean toExecute = false; try {
for (Dml dml : dmls) { boolean toExecute = false;
if (!toExecute) { for (Dml dml : dmls) {
toExecute = function.apply(dml); if (!toExecute) {
} else { toExecute = function.apply(dml);
function.apply(dml); } else {
function.apply(dml);
}
} }
} if (toExecute) {
if (toExecute) { List<Future> futures = new ArrayList<>();
List<Future> futures = new ArrayList<>(); for (int i = 0; i < threads; i++) {
for (int i = 0; i < threads; i++) { int j = i;
int j = i; futures.add(executorThreads[i].submit(() -> {
futures.add(executorThreads[i].submit(() -> { try {
dmlsPartition[j]
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
batchExecutors[j].rollback();
throw new RuntimeException(e);
}
}));
}
futures.forEach(future -> {
try { try {
dmlsPartition[j] future.get();
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); } catch (ExecutionException | InterruptedException e) {
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
batchExecutors[j].rollback();
throw new RuntimeException(e); throw new RuntimeException(e);
} }
})); });
} }
} finally {
futures.forEach(future -> { for (BatchExecutor batchExecutor : batchExecutors) {
try { if (batchExecutor != null) {
future.get(); batchExecutor.close();
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
} }
});
}
for (BatchExecutor batchExecutor : batchExecutors) {
if (batchExecutor != null) {
batchExecutor.close();
} }
} }
} }

8
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
View File

@@ -16,7 +16,7 @@ import com.google.common.collect.MigrateMap;
/** /**
* 使用自定义的ZooKeeperx for zk connection * 使用自定义的ZooKeeperx for zk connection
* *
* @author jianghang 2012-7-10 下午02:31:15 * @author jianghang 2012-7-10 下午02:31:15
* @version 1.0.0 * @version 1.0.0
*/ */
@@ -64,7 +64,7 @@ public class ZkClientx extends ZkClient {
/** /**
* Create a persistent Sequential node. * Create a persistent Sequential node.
* *
* @param path * @param path
* @param createParents if true all parent dirs are created as well and no * @param createParents if true all parent dirs are created as well and no
* {@link ZkNodeExistsException} is thrown in case the path already exists * {@link ZkNodeExistsException} is thrown in case the path already exists
@@ -92,7 +92,7 @@ public class ZkClientx extends ZkClient {
/** /**
* Create a persistent Sequential node. * Create a persistent Sequential node.
* *
* @param path * @param path
* @param data * @param data
* @param createParents if true all parent dirs are created as well and no * @param createParents if true all parent dirs are created as well and no
@@ -123,7 +123,7 @@ public class ZkClientx extends ZkClient {
/** /**
* Create a persistent Sequential node. * Create a persistent Sequential node.
* *
* @param path * @param path
* @param data * @param data
* @param createParents if true all parent dirs are created as well and no * @param createParents if true all parent dirs are created as well and no

153
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -68,6 +68,7 @@ public class CanalController {
private CanalInstanceGenerator instanceGenerator; private CanalInstanceGenerator instanceGenerator;
private ZkClientx zkclientx; private ZkClientx zkclientx;
private String zkServers;
private CanalMQStarter canalMQStarter; private CanalMQStarter canalMQStarter;
@@ -132,6 +133,7 @@ public class CanalController {
} }
final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS); final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS);
if (StringUtils.isNotEmpty(zkServers)) { if (StringUtils.isNotEmpty(zkServers)) {
this.zkServers = zkServers;
zkclientx = ZkClientx.getZkClient(zkServers); zkclientx = ZkClientx.getZkClient(zkServers);
// 初始化系统目录 // 初始化系统目录
zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true); zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true);
@@ -140,80 +142,81 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port); final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData); ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() { ServerRunningMonitors
.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) { public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination); runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() { runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter() { public void processActiveEnter() {
try { try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination); embededCanalServer.start(destination);
} finally { } finally {
MDC.remove(CanalConstants.MDC_DESTINATION); MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
} }
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
} }
}
public void processStop() { public void processActiveExit() {
try { try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) { embededCanalServer.stop(destination);
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":" } finally {
+ port); MDC.remove(CanalConstants.MDC_DESTINATION);
releaseCid(path);
} }
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
} }
}
}); public void processStart() {
if (zkclientx != null) { try {
runningMonitor.setZkClient(zkclientx); if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
} }
// 触发创建一下cid节点 }));
runningMonitor.init();
return runningMonitor;
}
}));
// 初始化monitor机制 // 初始化monitor机制
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
@@ -265,7 +268,8 @@ public class CanalController {
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() { instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode) { public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL)); int scanInterval = Integer
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) { if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -373,7 +377,8 @@ public class CanalController {
InstanceConfig oldConfig = instanceConfigs.put(destination, config); InstanceConfig oldConfig = instanceConfigs.put(destination, config);
if (oldConfig != null) { if (oldConfig != null) {
logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config); logger
.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
} }
} }
} }
@@ -478,6 +483,7 @@ public class CanalController {
} }
public void stop() throws Throwable { public void stop() throws Throwable {
if (canalServer != null) { if (canalServer != null) {
canalServer.stop(); canalServer.stop();
} }
@@ -502,6 +508,19 @@ public class CanalController {
if (zkclientx != null) { if (zkclientx != null) {
zkclientx.close(); zkclientx.close();
if (zkServers != null) {
ZkClientx.removeZkClient(zkServers);
}
}
if (instanceConfigs != null) {
instanceConfigs.clear();
}
if (managerClients != null) {
managerClients.clear();
}
if (instanceConfigMonitors != null) {
instanceConfigMonitors.clear();
} }
ZkClientx.clearClients(); ZkClientx.clearClients();

663
meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
View File

@@ -1,329 +1,334 @@
package com.alibaba.otter.canal.meta; package com.alibaba.otter.canal.meta;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.I0Itec.zkclient.exception.ZkNoNodeException; import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.I0Itec.zkclient.exception.ZkNodeExistsException; import org.I0Itec.zkclient.exception.ZkNodeExistsException;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.utils.JsonUtils; import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx; import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException; import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity; import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position; import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange; import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
/** /**
* zk 版本的 canal manager， 存储结构： * zk 版本的 canal manager， 存储结构：
* *
* <pre> * <pre>
* /otter * /otter
* canal * canal
* destinations * destinations
* dest1 * dest1
* client1 * client1
* filter * filter
* batch_mark * batch_mark
* 1 * 1
* 2 * 2
* 3 * 3
* </pre> * </pre>
* *
* @author zebin.xuzb @ 2012-6-21 * @author zebin.xuzb @ 2012-6-21
* @author jianghang * @author jianghang
* @version 1.0.0 * @version 1.0.0
*/ */
public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager { public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
private static final String ENCODE = "UTF-8"; private static final String ENCODE = "UTF-8";
private ZkClientx zkClientx; private ZkClientx zkClientx;
public void start() { public void start() {
super.start(); super.start();
Assert.notNull(zkClientx); Assert.notNull(zkClientx);
} }
public void stop() { public void stop() {
super.stop(); zkClientx = null;
} super.stop();
}
public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(), public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
clientIdentity.getClientId()); String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
clientIdentity.getClientId());
try {
zkClientx.createPersistent(path, true); try {
} catch (ZkNodeExistsException e) { zkClientx.createPersistent(path, true);
// ignore } catch (ZkNodeExistsException e) {
} // ignore
if (clientIdentity.hasFilter()) { }
String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(), if (clientIdentity.hasFilter()) {
clientIdentity.getClientId()); String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
byte[] bytes = null;
try { byte[] bytes = null;
bytes = clientIdentity.getFilter().getBytes(ENCODE); try {
} catch (UnsupportedEncodingException e) { bytes = clientIdentity.getFilter().getBytes(ENCODE);
throw new CanalMetaManagerException(e); } catch (UnsupportedEncodingException e) {
} throw new CanalMetaManagerException(e);
}
try {
zkClientx.createPersistent(filterPath, bytes); try {
} catch (ZkNodeExistsException e) { zkClientx.createPersistent(filterPath, bytes);
// ignore } catch (ZkNodeExistsException e) {
zkClientx.writeData(filterPath, bytes); // ignore
} zkClientx.writeData(filterPath, bytes);
} }
} }
}
public boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(), public boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
clientIdentity.getClientId()); String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
return zkClientx.exists(path); clientIdentity.getClientId());
} return zkClientx.exists(path);
}
public void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(), public void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
clientIdentity.getClientId()); String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
zkClientx.deleteRecursive(path); // 递归删除所有信息 clientIdentity.getClientId());
} zkClientx.deleteRecursive(path); // 递归删除所有信息
}
public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getDestinationPath(destination); public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
List<String> childs = null; if (zkClientx == null) {
try { return new ArrayList<ClientIdentity>();
childs = zkClientx.getChildren(path); }
} catch (ZkNoNodeException e) { String path = ZookeeperPathUtils.getDestinationPath(destination);
// ignore List<String> childs = null;
} try {
childs = zkClientx.getChildren(path);
if (CollectionUtils.isEmpty(childs)) { } catch (ZkNoNodeException e) {
return new ArrayList<ClientIdentity>(); // ignore
} }
List<Short> clientIds = new ArrayList<Short>();
for (String child : childs) { if (CollectionUtils.isEmpty(childs)) {
if (StringUtils.isNumeric(child)) { return new ArrayList<ClientIdentity>();
clientIds.add(ZookeeperPathUtils.getClientId(child)); }
} List<Short> clientIds = new ArrayList<Short>();
} for (String child : childs) {
if (StringUtils.isNumeric(child)) {
Collections.sort(clientIds); // 进行一个排序 clientIds.add(ZookeeperPathUtils.getClientId(child));
List<ClientIdentity> clientIdentities = Lists.newArrayList(); }
for (Short clientId : clientIds) { }
path = ZookeeperPathUtils.getFilterPath(destination, clientId);
byte[] bytes = zkClientx.readData(path, true); Collections.sort(clientIds); // 进行一个排序
String filter = null; List<ClientIdentity> clientIdentities = Lists.newArrayList();
if (bytes != null) { for (Short clientId : clientIds) {
try { path = ZookeeperPathUtils.getFilterPath(destination, clientId);
filter = new String(bytes, ENCODE); byte[] bytes = zkClientx.readData(path, true);
} catch (UnsupportedEncodingException e) { String filter = null;
throw new CanalMetaManagerException(e); if (bytes != null) {
} try {
} filter = new String(bytes, ENCODE);
clientIdentities.add(new ClientIdentity(destination, clientId, filter)); } catch (UnsupportedEncodingException e) {
} throw new CanalMetaManagerException(e);
}
return clientIdentities; }
} clientIdentities.add(new ClientIdentity(destination, clientId, filter));
}
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId()); return clientIdentities;
}
byte[] data = zkClientx.readData(path, true);
if (data == null || data.length == 0) { public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
return null; String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
}
byte[] data = zkClientx.readData(path, true);
return JsonUtils.unmarshalFromByte(data, Position.class); if (data == null || data.length == 0) {
} return null;
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId()); return JsonUtils.unmarshalFromByte(data, Position.class);
byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName); }
try {
zkClientx.writeData(path, data); public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
} catch (ZkNoNodeException e) { String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建 byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
} try {
} zkClientx.writeData(path, data);
} catch (ZkNoNodeException e) {
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException { zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); }
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName); }
String batchPath = zkClientx.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR,
data, public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
true); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR); clientIdentity.getClientId());
return ZookeeperPathUtils.getBatchMarkId(batchIdString); byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
} String batchPath = zkClientx
.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId) String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
throws CanalMetaManagerException { return ZookeeperPathUtils.getBatchMarkId(batchIdString);
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), }
clientIdentity.getClientId(),
batchId); public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName); Long batchId) throws CanalMetaManagerException {
zkClientx.createPersistent(path, data, true); String path = ZookeeperPathUtils
} .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException { zkClientx.createPersistent(path, data, true);
String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), }
clientIdentity.getClientId());
List<String> nodes = zkClientx.getChildren(batchsPath); public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
if (CollectionUtils.isEmpty(nodes)) { String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
// 没有batch记录 clientIdentity.getClientId());
return null; List<String> nodes = zkClientx.getChildren(batchsPath);
} if (CollectionUtils.isEmpty(nodes)) {
// 没有batch记录
// 找到最小的Id return null;
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); }
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString)); // 找到最小的Id
} ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
Long minBatchId = Collections.min(batchIds); for (String batchIdString : nodes) {
if (!minBatchId.equals(batchId)) { batchIds.add(Long.valueOf(batchIdString));
// 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据 }
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId)); Long minBatchId = Collections.min(batchIds);
} if (!minBatchId.equals(batchId)) {
// 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
if (!batchIds.contains(batchId)) { throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
// 不存在对应的batchId }
return null;
} if (!batchIds.contains(batchId)) {
PositionRange positionRange = getBatch(clientIdentity, batchId); // 不存在对应的batchId
if (positionRange != null) { return null;
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), }
clientIdentity.getClientId(), PositionRange positionRange = getBatch(clientIdentity, batchId);
batchId); if (positionRange != null) {
zkClientx.delete(path); String path = ZookeeperPathUtils
} .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
zkClientx.delete(path);
return positionRange; }
}
return positionRange;
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException { }
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(), public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
batchId); String path = ZookeeperPathUtils
byte[] data = zkClientx.readData(path, true); .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
if (data == null) { byte[] data = zkClientx.readData(path, true);
return null; if (data == null) {
} return null;
}
PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
return positionRange; PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
} return positionRange;
}
public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
List<String> batchChilds = zkClientx.getChildren(path); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
for (String batchChild : batchChilds) { List<String> batchChilds = zkClientx.getChildren(path);
String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
zkClientx.delete(batchPath); for (String batchChild : batchChilds) {
} String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
} zkClientx.delete(batchPath);
}
public PositionRange getLastestBatch(ClientIdentity clientIdentity) { }
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
List<String> nodes = null; public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
try { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
nodes = zkClientx.getChildren(path); clientIdentity.getClientId());
} catch (ZkNoNodeException e) { List<String> nodes = null;
// ignore try {
} nodes = zkClientx.getChildren(path);
} catch (ZkNoNodeException e) {
if (CollectionUtils.isEmpty(nodes)) { // ignore
return null; }
}
// 找到最大的Id if (CollectionUtils.isEmpty(nodes)) {
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); return null;
for (String batchIdString : nodes) { }
batchIds.add(Long.valueOf(batchIdString)); // 找到最大的Id
} ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
Long maxBatchId = Collections.max(batchIds); for (String batchIdString : nodes) {
PositionRange result = getBatch(clientIdentity, maxBatchId); batchIds.add(Long.valueOf(batchIdString));
if (result == null) { // 出现为null，说明zk节点有变化，重新获取 }
return getLastestBatch(clientIdentity); Long maxBatchId = Collections.max(batchIds);
} else { PositionRange result = getBatch(clientIdentity, maxBatchId);
return result; if (result == null) { // 出现为null，说明zk节点有变化，重新获取
} return getLastestBatch(clientIdentity);
} } else {
return result;
public PositionRange getFirstBatch(ClientIdentity clientIdentity) { }
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); }
List<String> nodes = null;
try { public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
nodes = zkClientx.getChildren(path); String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
} catch (ZkNoNodeException e) { clientIdentity.getClientId());
// ignore List<String> nodes = null;
} try {
nodes = zkClientx.getChildren(path);
if (CollectionUtils.isEmpty(nodes)) { } catch (ZkNoNodeException e) {
return null; // ignore
} }
// 找到最小的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); if (CollectionUtils.isEmpty(nodes)) {
for (String batchIdString : nodes) { return null;
batchIds.add(Long.valueOf(batchIdString)); }
} // 找到最小的Id
Long minBatchId = Collections.min(batchIds); ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
PositionRange result = getBatch(clientIdentity, minBatchId); for (String batchIdString : nodes) {
if (result == null) { // 出现为null，说明zk节点有变化，重新获取 batchIds.add(Long.valueOf(batchIdString));
return getFirstBatch(clientIdentity); }
} else { Long minBatchId = Collections.min(batchIds);
return result; PositionRange result = getBatch(clientIdentity, minBatchId);
} if (result == null) { // 出现为null，说明zk节点有变化，重新获取
} return getFirstBatch(clientIdentity);
} else {
public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) { return result;
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); }
List<String> nodes = null; }
try {
nodes = zkClientx.getChildren(path); public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
} catch (ZkNoNodeException e) { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
// ignore clientIdentity.getClientId());
} List<String> nodes = null;
try {
if (CollectionUtils.isEmpty(nodes)) { nodes = zkClientx.getChildren(path);
return Maps.newHashMap(); } catch (ZkNoNodeException e) {
} // ignore
// 找到最大的Id }
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
for (String batchIdString : nodes) { if (CollectionUtils.isEmpty(nodes)) {
batchIds.add(Long.valueOf(batchIdString)); return Maps.newHashMap();
} }
// 找到最大的Id
Collections.sort(batchIds); // 从小到大排序 ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap(); for (String batchIdString : nodes) {
for (Long batchId : batchIds) { batchIds.add(Long.valueOf(batchIdString));
PositionRange result = getBatch(clientIdentity, batchId); }
if (result == null) {// 出现为null，说明zk节点有变化，重新获取
return listAllBatchs(clientIdentity); Collections.sort(batchIds); // 从小到大排序
} else { Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap();
positionRanges.put(batchId, result); for (Long batchId : batchIds) {
} PositionRange result = getBatch(clientIdentity, batchId);
} if (result == null) {// 出现为null，说明zk节点有变化，重新获取
return listAllBatchs(clientIdentity);
return positionRanges; } else {
} positionRanges.put(batchId, result);
}
// =========== setter ========== }
public void setZkClientx(ZkClientx zkClientx) { return positionRanges;
this.zkClientx = zkClientx; }
}
// =========== setter ==========
}
public void setZkClientx(ZkClientx zkClientx) {
this.zkClientx = zkClientx;
}
}