chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed binlog seek & aliyun ak/sk

Browse Source
This commit is contained in:
agapple
2018-11-21 13:51:34 +08:00
parent c0bbe530f1
commit 638952fad1
9 changed files with 111 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
View File

@@ -109,11 +109,6 @@ public final class LogDecoder {
try { try {
/* Decoding binary-log to event */ /* Decoding binary-log to event */
event = decode(buffer, header, context); event = decode(buffer, header, context);
if (event != null) {
// set logFileName
event.getHeader().setLogFileName(context.getLogPosition().getFileName());
event.setSemival(buffer.semival);
}
} catch (IOException e) { } catch (IOException e) {
if (logger.isWarnEnabled()) { if (logger.isWarnEnabled()) {
logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: " logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: "
@@ -128,6 +123,12 @@ public final class LogDecoder {
event = new UnknownLogEvent(header); event = new UnknownLogEvent(header);
} }
if (event != null) {
// set logFileName
event.getHeader().setLogFileName(context.getLogPosition().getFileName());
event.setSemival(buffer.semival);
}
/* consume this binary-log. */ /* consume this binary-log. */
buffer.consume(len); buffer.consume(len);
return event; return event;

2
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
View File

@@ -47,6 +47,8 @@ public class CanalConstants {
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
public static String getInstanceModeKey(String destination) { public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

156
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -17,7 +17,6 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.alibaba.otter.canal.common.CanalException; import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.AddressUtils; import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx; import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
@@ -96,6 +95,16 @@ public class CanalController {
System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel); System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel);
} }
// 兼容1.1.0版本的ak/sk参数名
String accesskey = getProperty(properties, "canal.instance.rds.accesskey");
String secretkey = getProperty(properties, "canal.instance.rds.secretkey");
if (StringUtils.isNotEmpty(accesskey)) {
System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey);
}
if (StringUtils.isNotEmpty(secretkey)) {
System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey);
}
// 准备canal server // 准备canal server
cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID)); cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
ip = getProperty(properties, CanalConstants.CANAL_IP); ip = getProperty(properties, CanalConstants.CANAL_IP);
@@ -131,81 +140,80 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port); final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData); ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) { public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination); runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() { runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter() { public void processActiveEnter() {
try { try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination); embededCanalServer.start(destination);
} finally { } finally {
MDC.remove(CanalConstants.MDC_DESTINATION); MDC.remove(CanalConstants.MDC_DESTINATION);
}
} }
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
ip + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
} }
// 触发创建一下cid节点
runningMonitor.init(); public void processActiveExit() {
return runningMonitor; try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
} }
})); // 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
}
}));
// 初始化monitor机制 // 初始化monitor机制
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
@@ -257,8 +265,7 @@ public class CanalController {
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() { instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode) { public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) { if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -366,8 +373,7 @@ public class CanalController {
InstanceConfig oldConfig = instanceConfigs.put(destination, config); InstanceConfig oldConfig = instanceConfigs.put(destination, config);
if (oldConfig != null) { if (oldConfig != null) {
logger logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
} }
} }
} }

9
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -125,6 +125,15 @@ public class CanalLauncher {
if (!StringUtils.isEmpty(acks)) { if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks); mqProperties.setAcks(acks);
} }
String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunSecretKey(aliyunSecretKey);
}
return mqProperties; return mqProperties;
} }

10
deployer/src/main/resources/canal.properties
View File

@@ -74,9 +74,9 @@ canal.instance.tsdb.snapshot.interval=24
# purge snapshot expire , default 360 hour(15 days) # purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire=360 canal.instance.tsdb.snapshot.expire=360
# rds oss binlog account # aliyun ak/sk , support rds/mq
canal.instance.rds.accesskey = canal.aliyun.accesskey=
canal.instance.rds.secretkey = canal.aliyun.secretkey=
################################################# #################################################
######### destinations ############# ######### destinations #############
@@ -109,5 +109,5 @@ canal.mq.bufferMemory=33554432
canal.mq.canalBatchSize=50 canal.mq.canalBatchSize=50
canal.mq.canalGetTimeout=100 canal.mq.canalGetTimeout=100
canal.mq.flatMessage=true canal.mq.flatMessage=true
canal.mq.compressionType= canal.mq.compressionType=none
canal.mq.acks= canal.mq.acks=all

4
deployer/src/main/resources/spring/base-instance.xml
View File

@@ -32,8 +32,8 @@
</bean> </bean>
<bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true"> <bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true">
<property name="accesskey" value="${canal.instance.rds.accesskey:}" /> <property name="accesskey" value="${canal.aliyun.accesskey:}" />
<property name="secretkey" value="${canal.instance.rds.secretkey:}" /> <property name="secretkey" value="${canal.aliyun.secretkey:}" />
<property name="instanceId" value="${canal.instance.rds.instanceId:}" /> <property name="instanceId" value="${canal.instance.rds.instanceId:}" />
</bean> </bean>
</beans> </beans>

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -747,7 +747,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) { if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) {
// 初始位点 // 初始位点
entryPosition = new EntryPosition(searchBinlogFile, entryPosition = new EntryPosition(searchBinlogFile,
event.getLogPos(), event.getLogPos() - event.getEventLen(),
event.getWhen() * 1000, event.getWhen() * 1000,
event.getServerId()); event.getServerId());
logPosition.setPostion(entryPosition); logPosition.setPostion(entryPosition);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -657,7 +657,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
+ tableMeta.getFullName() + ", db fieldMeta : " + tableMeta.getFullName() + ", db fieldMeta : "
+ fieldMeta.toString() + " , binlog fieldMeta : " + info.toString() + fieldMeta.toString() + " , binlog fieldMeta : " + info.toString()
+ " , on : " + event.getHeader().getLogFileName() + ":" + " , on : " + event.getHeader().getLogFileName() + ":"
+ event.getHeader().getLogPos()); + (event.getHeader().getLogPos() - event.getHeader().getEventLen()));
} }
} }

8
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -20,10 +20,10 @@ public class MQProperties {
private int canalBatchSize = 50; private int canalBatchSize = 50;
private Long canalGetTimeout = 100L; private Long canalGetTimeout = 100L;
private boolean flatMessage = true; private boolean flatMessage = true;
private String compressionType = "none"; private String compressionType = "none";
private String acks = "all"; private String acks = "all";
private String aliyunAccessKey = ""; private String aliyunAccessKey = "";
private String aliyunSecretKey = ""; private String aliyunSecretKey = "";
public static class CanalDestination { public static class CanalDestination {