chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23 from alibaba/master

Browse Source 
merge
This commit is contained in:
rewerma
2018-11-22 09:04:24 +08:00
committed by GitHub
parent e711f1a091 aa89453bf6
commit 3b25b3b342
19 changed files with 259 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -12,25 +12,26 @@ import java.util.Map;
*/ */
public class CanalClientConfig { public class CanalClientConfig {
private String canalServerHost; // 单机模式下canal server的 ip:port // 单机模式下canal server的ip:port
private String canalServerHost;
private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!! // 集群模式下的zk地址,如果配置了单机地址则以单机为准!!
private String zookeeperHosts;
private String mqServers; // kafka or rocket mq 地址 // kafka or rocket mq 地址
private String mqServers;
private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式 // 是否已flatMessage模式传输,只适用于mq模式
private Boolean flatMessage = true;
private Integer batchSize; // 批大小 // 批大小
private Integer batchSize;
private Integer syncBatchSize = 1000; // 同步分批提交大小 // 同步分批提交大小
private Integer syncBatchSize = 1000;
private Integer retries; // 重试次数 // 重试次数
private Integer retries;
private Long timeout; // 消费超时时间 // 消费超时时间
private Long timeout;
private String mode = "tcp"; // 模式 tcp kafka rocketMQ // 模式 tcp kafka rocketMQ
private String mode = "tcp";
private List<CanalAdapter> canalAdapters; // canal adapters 配置 // canal adapters 配置
private List<CanalAdapter> canalAdapters;
public String getCanalServerHost() { public String getCanalServerHost() {
return canalServerHost; return canalServerHost;
@@ -116,8 +117,6 @@ public class CanalClientConfig {
private String instance; // 实例名 private String instance; // 实例名
private String topic; // mq topic
private List<Group> groups; // 适配器分组列表 private List<Group> groups; // 适配器分组列表
public String getInstance() { public String getInstance() {
@@ -137,22 +136,13 @@ public class CanalClientConfig {
public void setGroups(List<Group> groups) { public void setGroups(List<Group> groups) {
this.groups = groups; this.groups = groups;
} }
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
} }
public static class Group { public static class Group {
private String groupId; // group id // group id
private String groupId = "default";
private List<OuterAdapterConfig> outerAdapters; // 适配器列表 private List<OuterAdapterConfig> outerAdapters; // 适配器列表
private Map<String, OuterAdapterConfig> outerAdaptersMap = new LinkedHashMap<>(); private Map<String, OuterAdapterConfig> outerAdaptersMap = new LinkedHashMap<>();
public String getGroupId() { public String getGroupId() {

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterCanalConfig.java
View File

@@ -37,8 +37,6 @@ public class AdapterCanalConfig extends CanalClientConfig {
for (CanalAdapter canalAdapter : canalAdapters) { for (CanalAdapter canalAdapter : canalAdapters) {
if (canalAdapter.getInstance() != null) { if (canalAdapter.getInstance() != null) {
DESTINATIONS.add(canalAdapter.getInstance()); DESTINATIONS.add(canalAdapter.getInstance());
} else if (canalAdapter.getTopic() != null) {
DESTINATIONS.add(canalAdapter.getInstance());
} }
} }
} }

12
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,9 +2,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -12,6 +15,7 @@ import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.CanalMQConnector; import com.alibaba.otter.canal.client.CanalMQConnector;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil; import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
@@ -180,6 +184,9 @@ public abstract class AbstractCanalAdapterWorker {
*/ */
private void batchSync(List<Dml> dmls, OuterAdapter adapter) { private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
// 分批同步 // 分批同步
if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
adapter.sync(dmls);
} else {
int len = 0; int len = 0;
List<Dml> dmlsBatch = new ArrayList<>(); List<Dml> dmlsBatch = new ArrayList<>();
for (Dml dml : dmls) { for (Dml dml : dmls) {
@@ -193,6 +200,7 @@ public abstract class AbstractCanalAdapterWorker {
} }
adapter.sync(dmlsBatch); adapter.sync(dmlsBatch);
} }
}
public void start() { public void start() {
if (!running) { if (!running) {

16
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -97,14 +97,14 @@ public class CanalAdapterLoader {
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
canalClientConfig.getMqServers(), canalClientConfig.getMqServers(),
canalAdapter.getTopic(), canalAdapter.getInstance(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,
canalClientConfig.getFlatMessage()); canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker); canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start(); canalKafkaWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed", logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
canalAdapter.getTopic() + "-" + group.getGroupId()); + group.getGroupId());
} }
} }
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
@@ -119,15 +119,15 @@ public class CanalAdapterLoader {
canalOuterAdapterGroups.add(canalOuterAdapters); canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getMqServers(), canalClientConfig.getMqServers(),
canalAdapter.getTopic(), canalAdapter.getInstance(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,
canalClientConfig.getFlatMessage()); canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start(); rocketMQWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed", logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
canalAdapter.getTopic() + "-" + group.getGroupId()); + group.getGroupId());
} }
} }
} }

5
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -28,10 +28,9 @@ canal.conf:
# username: root # username: root
# password: 121212 # password: 121212
canalAdapters: canalAdapters:
- instance: example - instance: example # canal instance Name or mq topic name
# topic: example
groups: groups:
- #groupId: g1 - groupId: g1
outerAdapters: outerAdapters:
- name: logger - name: logger
# - name: rdb # - name: rdb

7
client/pom.xml
View File

@@ -101,12 +101,9 @@
<version>${spring_version}</version> <version>${spring_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- 客户端要使用请单独引入rocketmq-client依赖 -->
<dependency> <dependency>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId> <artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
<scope>provided</scope>
</dependency> </dependency>
<!-- 客户端要使用请单独引入kafka-clients依赖 --> <!-- 客户端要使用请单独引入kafka-clients依赖 -->
<dependency> <dependency>
@@ -122,6 +119,10 @@
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>com.aliyun.openservices</groupId>
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>

26
client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java
View File

@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.client.rocketmq; package com.alibaba.otter.canal.client.rocketmq;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import java.util.List; import java.util.List;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
@@ -10,8 +12,10 @@ import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -47,6 +51,9 @@ public class RocketMQCanalConnector implements CanalMQConnector {
private long batchProcessTimeout = 60 * 1000; private long batchProcessTimeout = 60 * 1000;
private boolean flatMessage; private boolean flatMessage;
private volatile ConsumerBatchMessage lastGetBatchMessage = null; private volatile ConsumerBatchMessage lastGetBatchMessage = null;
private String accessKey;
private String secretKey;
public RocketMQCanalConnector(String nameServer, String topic, String groupName, boolean flatMessage){ public RocketMQCanalConnector(String nameServer, String topic, String groupName, boolean flatMessage){
this.nameServer = nameServer; this.nameServer = nameServer;
@@ -56,8 +63,25 @@ public class RocketMQCanalConnector implements CanalMQConnector {
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024); this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
} }
public RocketMQCanalConnector(String nameServer, String topic, String groupName,
String accessKey, String secretKey, boolean flatMessage){
this(nameServer, topic, groupName, flatMessage);
this.accessKey = accessKey;
this.secretKey = secretKey;
}
public void connect() throws CanalClientException { public void connect() throws CanalClientException {
rocketMQConsumer = new DefaultMQPushConsumer(groupName);
RPCHook rpcHook = null;
if(null != accessKey && accessKey.length() > 0
&& null != secretKey && secretKey.length() > 0){
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(accessKey);
sessionCredentials.setSecretKey(secretKey);
rpcHook = new ClientRPCHook(sessionCredentials);
}
rocketMQConsumer = new DefaultMQPushConsumer(groupName, rpcHook, new AllocateMessageQueueAveragely());
rocketMQConsumer.setVipChannelEnabled(false);
if (!StringUtils.isBlank(nameServer)) { if (!StringUtils.isBlank(nameServer)) {
rocketMQConsumer.setNamesrvAddr(nameServer); rocketMQConsumer.setNamesrvAddr(nameServer);
} }

11
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
View File

@@ -109,11 +109,6 @@ public final class LogDecoder {
try { try {
/* Decoding binary-log to event */ /* Decoding binary-log to event */
event = decode(buffer, header, context); event = decode(buffer, header, context);
if (event != null) {
// set logFileName
event.getHeader().setLogFileName(context.getLogPosition().getFileName());
event.setSemival(buffer.semival);
}
} catch (IOException e) { } catch (IOException e) {
if (logger.isWarnEnabled()) { if (logger.isWarnEnabled()) {
logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: " logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: "
@@ -128,6 +123,12 @@ public final class LogDecoder {
event = new UnknownLogEvent(header); event = new UnknownLogEvent(header);
} }
if (event != null) {
// set logFileName
event.getHeader().setLogFileName(context.getLogPosition().getFileName());
event.setSemival(buffer.semival);
}
/* consume this binary-log. */ /* consume this binary-log. */
buffer.consume(len); buffer.consume(len);
return event; return event;

2
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
View File

@@ -47,6 +47,8 @@ public class CanalConstants {
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage"; public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType"; public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks"; public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
public static String getInstanceModeKey(String destination) { public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

28
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -17,7 +17,6 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.alibaba.otter.canal.common.CanalException; import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.common.utils.AddressUtils; import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx; import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
@@ -96,6 +95,16 @@ public class CanalController {
System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel); System.setProperty(CanalConstants.CANAL_SOCKETCHANNEL, socketChannel);
} }
// 兼容1.1.0版本的ak/sk参数名
String accesskey = getProperty(properties, "canal.instance.rds.accesskey");
String secretkey = getProperty(properties, "canal.instance.rds.secretkey");
if (StringUtils.isNotEmpty(accesskey)) {
System.setProperty(CanalConstants.CANAL_ALIYUN_ACCESSKEY, accesskey);
}
if (StringUtils.isNotEmpty(secretkey)) {
System.setProperty(CanalConstants.CANAL_ALIYUN_SECRETKEY, secretkey);
}
// 准备canal server // 准备canal server
cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID)); cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
ip = getProperty(properties, CanalConstants.CANAL_IP); ip = getProperty(properties, CanalConstants.CANAL_IP);
@@ -131,8 +140,7 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port); final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData); ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) { public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
@@ -160,8 +168,8 @@ public class CanalController {
public void processStart() { public void processStart() {
try { try {
if (zkclientx != null) { if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
ip + ":" + port); + port);
initCid(path); initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() { zkclientx.subscribeStateChanges(new IZkStateListener() {
@@ -188,8 +196,8 @@ public class CanalController {
try { try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) { if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
ip + ":" + port); + port);
releaseCid(path); releaseCid(path);
} }
} finally { } finally {
@@ -257,8 +265,7 @@ public class CanalController {
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() { instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode) { public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) { if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
@@ -366,8 +373,7 @@ public class CanalController {
InstanceConfig oldConfig = instanceConfigs.put(destination, config); InstanceConfig oldConfig = instanceConfigs.put(destination, config);
if (oldConfig != null) { if (oldConfig != null) {
logger logger.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
.warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
} }
} }
} }

9
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -125,6 +125,15 @@ public class CanalLauncher {
if (!StringUtils.isEmpty(acks)) { if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks); mqProperties.setAcks(acks);
} }
String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunSecretKey(aliyunSecretKey);
}
return mqProperties; return mqProperties;
} }

56
deployer/src/main/resources/canal.properties
View File

@@ -1,11 +1,11 @@
################################################# #################################################
######### common argument ############# ######### common argument #############
################################################# #################################################
canal.id= 1 canal.id = 1
canal.ip= canal.ip =
canal.port=11111 canal.port = 11111
canal.metrics.pull.port=11112 canal.metrics.pull.port = 11112
canal.zkServers= canal.zkServers =
# flush data to zk # flush data to zk
canal.zookeeper.flush.period = 1000 canal.zookeeper.flush.period = 1000
canal.withoutNetty = false canal.withoutNetty = false
@@ -64,32 +64,32 @@ canal.instance.parser.parallel = true
canal.instance.parser.parallelBufferSize = 256 canal.instance.parser.parallelBufferSize = 256
# table meta tsdb info # table meta tsdb info
canal.instance.tsdb.enable=true canal.instance.tsdb.enable = true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername=canal canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword=canal canal.instance.tsdb.dbPassword = canal
# dump snapshot interval, default 24 hour # dump snapshot interval, default 24 hour
canal.instance.tsdb.snapshot.interval=24 canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days) # purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire=360 canal.instance.tsdb.snapshot.expire = 360
# rds oss binlog account # aliyun ak/sk , support rds/mq
canal.instance.rds.accesskey = canal.aliyun.accesskey =
canal.instance.rds.secretkey = canal.aliyun.secretkey =
################################################# #################################################
######### destinations ############# ######### destinations #############
################################################# #################################################
canal.destinations= example canal.destinations = example
# conf root dir # conf root dir
canal.conf.dir = ../conf canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance # auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true canal.auto.scan = true
canal.auto.scan.interval = 5 canal.auto.scan.interval = 5
canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
#canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml #canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
canal.instance.global.mode = spring canal.instance.global.mode = spring
canal.instance.global.lazy = false canal.instance.global.lazy = false
@@ -101,13 +101,13 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
################################################## ##################################################
######### MQ ############# ######### MQ #############
################################################## ##################################################
canal.mq.servers=127.0.0.1:6667 canal.mq.servers = 127.0.0.1:6667
canal.mq.retries=0 canal.mq.retries = 0
canal.mq.batchSize=16384 canal.mq.batchSize = 16384
canal.mq.lingerMs=1 canal.mq.lingerMs = 1
canal.mq.bufferMemory=33554432 canal.mq.bufferMemory = 33554432
canal.mq.canalBatchSize=50 canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout=100 canal.mq.canalGetTimeout = 100
canal.mq.flatMessage=true canal.mq.flatMessage = true
canal.mq.compressionType= canal.mq.compressionType = none
canal.mq.acks= canal.mq.acks = all

4
deployer/src/main/resources/spring/base-instance.xml
View File

@@ -32,8 +32,8 @@
</bean> </bean>
<bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true"> <bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true">
<property name="accesskey" value="${canal.instance.rds.accesskey:}" /> <property name="accesskey" value="${canal.aliyun.accesskey:}" />
<property name="secretkey" value="${canal.instance.rds.secretkey:}" /> <property name="secretkey" value="${canal.aliyun.secretkey:}" />
<property name="instanceId" value="${canal.instance.rds.instanceId:}" /> <property name="instanceId" value="${canal.instance.rds.instanceId:}" />
</bean> </bean>
</beans> </beans>

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -747,7 +747,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) { if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) {
// 初始位点 // 初始位点
entryPosition = new EntryPosition(searchBinlogFile, entryPosition = new EntryPosition(searchBinlogFile,
event.getLogPos(), event.getLogPos() - event.getEventLen(),
event.getWhen() * 1000, event.getWhen() * 1000,
event.getServerId()); event.getServerId());
logPosition.setPostion(entryPosition); logPosition.setPostion(entryPosition);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -657,7 +657,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
+ tableMeta.getFullName() + ", db fieldMeta : " + tableMeta.getFullName() + ", db fieldMeta : "
+ fieldMeta.toString() + " , binlog fieldMeta : " + info.toString() + fieldMeta.toString() + " , binlog fieldMeta : " + info.toString()
+ " , on : " + event.getHeader().getLogFileName() + ":" + " , on : " + event.getHeader().getLogFileName() + ":"
+ event.getHeader().getLogPos()); + (event.getHeader().getLogPos() - event.getHeader().getEventLen()));
} }
} }

10
pom.xml
View File

@@ -302,6 +302,16 @@
<artifactId>jsr305</artifactId> <artifactId>jsr305</artifactId>
<version>3.0.2</version> <version>3.0.2</version>
</dependency> </dependency>
<dependency>
<groupId>com.aliyun.openservices</groupId>
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>

5
server/pom.xml
View File

@@ -40,7 +40,6 @@
<dependency> <dependency>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId> <artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
</dependency> </dependency>
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 --> <!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
<dependency> <dependency>
@@ -55,5 +54,9 @@
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>com.aliyun.openservices</groupId>
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
</dependency>
</dependencies> </dependencies>
</project> </project>

17
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -22,6 +22,8 @@ public class MQProperties {
private boolean flatMessage = true; private boolean flatMessage = true;
private String compressionType = "none"; private String compressionType = "none";
private String acks = "all"; private String acks = "all";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
public static class CanalDestination { public static class CanalDestination {
@@ -168,4 +170,19 @@ public class MQProperties {
this.acks = acks; this.acks = acks;
} }
public String getAliyunAccessKey() {
return aliyunAccessKey;
}
public void setAliyunAccessKey(String aliyunAccessKey) {
this.aliyunAccessKey = aliyunAccessKey;
}
public String getAliyunSecretKey() {
return aliyunSecretKey;
}
public void setAliyunSecretKey(String aliyunSecretKey) {
this.aliyunSecretKey = aliyunSecretKey;
}
} }

16
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
View File

@@ -7,12 +7,15 @@ import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.server.exception.CanalServerException; import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer; import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -30,10 +33,19 @@ public class CanalRocketMQProducer implements CanalMQProducer {
@Override @Override
public void init(MQProperties rocketMQProperties) { public void init(MQProperties rocketMQProperties) {
this.mqProperties = rocketMQProperties; this.mqProperties = rocketMQProperties;
defaultMQProducer = new DefaultMQProducer(); RPCHook rpcHook = null;
if(rocketMQProperties.getAliyunAccessKey().length() > 0
&& rocketMQProperties.getAliyunSecretKey().length() > 0){
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
rpcHook = new ClientRPCHook(sessionCredentials);
}
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(), rpcHook);
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers()); defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
defaultMQProducer.setProducerGroup(rocketMQProperties.getProducerGroup());
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries()); defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
defaultMQProducer.setVipChannelEnabled(false);
logger.info("##Start RocketMQ producer##"); logger.info("##Start RocketMQ producer##");
try { try {
defaultMQProducer.start(); defaultMQProducer.start();