chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed #1430

Browse Source
This commit is contained in:
mcy
2019-01-17 10:22:08 +08:00
parent 739f3b4fa1
commit e387cfdee5
4 changed files with 462 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -88,44 +88,46 @@ public class RdbSyncService {
* @param function 回调方法
*/
public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
boolean toExecute = false;
for (Dml dml : dmls) {
if (!toExecute) {
toExecute = function.apply(dml);
} else {
function.apply(dml);
try {
boolean toExecute = false;
for (Dml dml : dmls) {
if (!toExecute) {
toExecute = function.apply(dml);
} else {
function.apply(dml);
}
}
}
if (toExecute) {
List<Future> futures = new ArrayList<>();
for (int i = 0; i < threads; i++) {
int j = i;
futures.add(executorThreads[i].submit(() -> {
if (toExecute) {
List<Future> futures = new ArrayList<>();
for (int i = 0; i < threads; i++) {
int j = i;
futures.add(executorThreads[i].submit(() -> {
try {
dmlsPartition[j]
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
batchExecutors[j].rollback();
throw new RuntimeException(e);
}
}));
}
futures.forEach(future -> {
try {
dmlsPartition[j]
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
batchExecutors[j].rollback();
future.get();
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
}
}));
});
}
futures.forEach(future -> {
try {
future.get();
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
} finally {
for (BatchExecutor batchExecutor : batchExecutors) {
if (batchExecutor != null) {
batchExecutor.close();
}
});
}
for (BatchExecutor batchExecutor : batchExecutors) {
if (batchExecutor != null) {
batchExecutor.close();
}
}
}

4
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
View File

@@ -34,6 +34,10 @@ public class ZkClientx extends ZkClient {
return clients.get(servers);
}
public static ZkClientx removeZkClient(String servers) {
return clients.remove(servers);
}
public ZkClientx(String serverstring){
this(serverstring, Integer.MAX_VALUE);
}

153
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -68,6 +68,7 @@ public class CanalController {
private CanalInstanceGenerator instanceGenerator;
private ZkClientx zkclientx;
private String zkServers;
private CanalMQStarter canalMQStarter;
@@ -132,6 +133,7 @@ public class CanalController {
}
final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS);
if (StringUtils.isNotEmpty(zkServers)) {
this.zkServers = zkServers;
zkclientx = ZkClientx.getZkClient(zkServers);
// 初始化系统目录
zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true);
@@ -140,80 +142,81 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
ServerRunningMonitors
.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
releaseCid(path);
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
}
}));
}));
// 初始化monitor机制
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
@@ -265,7 +268,8 @@ public class CanalController {
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
int scanInterval = Integer
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -373,7 +377,8 @@ public class CanalController {
InstanceConfig oldConfig = instanceConfigs.put(destination, config);
if (oldConfig != null) {
logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
logger
.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
}
}
}
@@ -478,6 +483,7 @@ public class CanalController {
}
public void stop() throws Throwable {
if (canalServer != null) {
canalServer.stop();
}
@@ -502,6 +508,19 @@ public class CanalController {
if (zkclientx != null) {
zkclientx.close();
if (zkServers != null) {
ZkClientx.removeZkClient(zkServers);
}
}
if (instanceConfigs != null) {
instanceConfigs.clear();
}
if (managerClients != null) {
managerClients.clear();
}
if (instanceConfigMonitors != null) {
instanceConfigMonitors.clear();
}
}

43
meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
View File

@@ -56,6 +56,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public void stop() {
zkClientx = null;
super.stop();
}
@@ -101,6 +102,9 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
if (zkClientx == null) {
return new ArrayList<ClientIdentity>();
}
String path = ZookeeperPathUtils.getDestinationPath(destination);
List<String> childs = null;
try {
@@ -160,20 +164,19 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
String batchPath = zkClientx.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR,
data,
true);
String batchPath = zkClientx
.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(),
batchId);
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
Long batchId) throws CanalMetaManagerException {
String path = ZookeeperPathUtils
.getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
zkClientx.createPersistent(path, data, true);
}
@@ -204,9 +207,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
PositionRange positionRange = getBatch(clientIdentity, batchId);
if (positionRange != null) {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(),
batchId);
String path = ZookeeperPathUtils
.getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
zkClientx.delete(path);
}
@@ -214,9 +216,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
clientIdentity.getClientId(),
batchId);
String path = ZookeeperPathUtils
.getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
byte[] data = zkClientx.readData(path, true);
if (data == null) {
return null;
@@ -227,7 +228,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> batchChilds = zkClientx.getChildren(path);
for (String batchChild : batchChilds) {
@@ -237,7 +239,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = null;
try {
nodes = zkClientx.getChildren(path);
@@ -263,7 +266,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = null;
try {
nodes = zkClientx.getChildren(path);
@@ -289,7 +293,8 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
clientIdentity.getClientId());
List<String> nodes = null;
try {
nodes = zkClientx.getChildren(path);