chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rds binlog 下载支持

Browse Source
This commit is contained in:
charles.lin
2018-08-08 19:29:52 +08:00
parent c6eee47af6
commit 61c7f7738d
131 changed files with 7250 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
deployer/pom.xml
View File

@@ -16,6 +16,14 @@
<artifactId>canal.server</artifactId>
<version>${project.version}</version>
</dependency>
<!-- 这里指定runtime的metrics provider-->
<!--<dependency>-->
<!--<groupId>com.alibaba.otter</groupId>-->
<!--<artifactId>canal.prometheus</artifactId>-->
<!--<version>${project.version}</version>-->
<!--<scope>runtime</scope>-->
<!--</dependency>-->
</dependencies>
<build>

15
deployer/src/main/bin/metrics_env.sh Normal file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
# Additional line arg for current prometheus solution
case "`uname`" in
Linux)
bin_abs_path=$(readlink -f $(dirname $0))
;;
*)
bin_abs_path=`cd $(dirname $0); pwd`
;;
esac
base=${bin_abs_path}/..
if [ $(ls $base/lib/aspectjweaver*.jar | wc -l) -eq 1 ]; then
WEAVER=$(ls $base/lib/aspectjweaver*.jar)
METRICS_OPTS=" -javaagent:"${WEAVER}" "
fi

7
deployer/src/main/bin/startup.sh
View File

@@ -94,7 +94,12 @@ then
echo LOG CONFIGURATION : $logback_configurationFile
echo canal conf : $canal_conf
echo CLASSPATH :$CLASSPATH
$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 &
# metrics support options
# if [ -x $base/bin/metrics_env.sh ]; then
# . $base/bin/metrics_env.sh
# echo METRICS_OPTS $METRICS_OPTS
# fi
$JAVA $JAVA_OPTS $METRICS_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 &
echo $! > $base/bin/canal.pid
echo "cd to $current_path for continue"

16
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -34,6 +34,7 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
@@ -303,7 +304,7 @@ public class CanalController {
return instanceGenerator.generate(destination);
} else if (config.getMode().isSpring()) {
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
synchronized (this) {
synchronized (CanalEventParser.class) {
try {
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination);
@@ -379,7 +380,18 @@ public class CanalController {
}
private String getProperty(Properties properties, String key) {
return StringUtils.trim(properties.getProperty(StringUtils.trim(key)));
key = StringUtils.trim(key);
String value = System.getProperty(key);
if (value == null) {
value = System.getenv(key);
}
if (value == null) {
value = properties.getProperty(key);
}
return StringUtils.trim(value);
}
public void start() throws Throwable {

10
deployer/src/main/resources/canal.properties
View File

@@ -3,7 +3,7 @@
#################################################
canal.id= 1
canal.ip=
canal.port= 11111
canal.port=11111
canal.zkServers=
# flush data to zk
canal.zookeeper.flush.period = 1000
@@ -43,6 +43,7 @@ canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
# binlog format/image check
canal.instance.binlog.format = ROW,STATEMENT,MIXED
@@ -58,6 +59,13 @@ canal.instance.parser.parallel = true
## disruptor ringbuffer size, must be power of 2
canal.instance.parser.parallelBufferSize = 256
# table meta tsdb info
canal.instance.tsdb.enable=true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername=canal
canal.instance.tsdb.dbPassword=canal
#################################################
######### destinations #############
#################################################

19
deployer/src/main/resources/example/instance.properties
View File

@@ -1,11 +1,12 @@
#################################################
## mysql serverId
canal.instance.mysql.slaveId=0
## mysql serverId , v1.0.26+ will autoGen
# canal.instance.mysql.slaveId=0
# enable gtid use true/false
canal.instance.gtidon=false
# position info
canal.instance.master.address=127.0.0.1:3306
# enable gtid use true/false
canal.instance.gtidon=false
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
@@ -13,23 +14,21 @@ canal.instance.master.gtid=
# table meta tsdb info
canal.instance.tsdb.enable=true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
canal.instance.tsdb.dbUsername=canal
canal.instance.tsdb.dbPassword=canal
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal
#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.defaultDatabaseName=test
canal.instance.connectionCharset=UTF-8
# table regex
canal.instance.filter.regex=.*\\..*
# table black regex

5
deployer/src/main/resources/spring/default-instance.xml
View File

@@ -81,6 +81,7 @@
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
@@ -148,7 +149,7 @@
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
<property name="standbyInfo">
@@ -156,7 +157,7 @@
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>

5
deployer/src/main/resources/spring/file-instance.xml
View File

@@ -67,6 +67,7 @@
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
@@ -133,7 +134,7 @@
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
<property name="standbyInfo">
@@ -141,7 +142,7 @@
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>

9
deployer/src/main/resources/spring/group-instance.xml
View File

@@ -64,6 +64,7 @@
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.group.GroupEventParser">
@@ -130,7 +131,7 @@
<property name="address" value="${canal.instance.master1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
<property name="standbyInfo">
@@ -138,7 +139,7 @@
<property name="address" value="${canal.instance.standby1.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
@@ -228,7 +229,7 @@
<property name="address" value="${canal.instance.master2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
<property name="standbyInfo">
@@ -236,7 +237,7 @@
<property name="address" value="${canal.instance.standby2.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>

3
deployer/src/main/resources/spring/local-instance.xml
View File

@@ -67,6 +67,7 @@
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser">
@@ -113,7 +114,7 @@
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>

5
deployer/src/main/resources/spring/memory-instance.xml
View File

@@ -64,6 +64,7 @@
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
@@ -121,7 +122,7 @@
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
<property name="standbyInfo">
@@ -129,7 +130,7 @@
<property name="address" value="${canal.instance.standby.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>