chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed rds binlog local

Browse Source
This commit is contained in:
agapple
2017-10-15 15:55:30 +08:00
parent e01e48ff65
commit 77342663e3
30 changed files with 1174 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java
View File

@@ -208,8 +208,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
action.reload(destination);
logger.info("auto notify reload {} successful.", destination);
} catch (Throwable e) {
logger.error(String.format("scan reload found[%s] but reload failed",
destination), e);
logger.error(String.format("scan reload found[%s] but reload failed", destination), e);
}
}
}
@@ -312,5 +311,4 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
}
}
}

1
deployer/src/main/resources/canal.properties
View File

@@ -66,6 +66,7 @@ canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml
canal.instance.global.mode = spring
canal.instance.global.lazy = false
#canal.instance.global.manager.address = 127.0.0.1:1099
#canal.instance.global.spring.xml = classpath:spring/local-instance.xml
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml

10
deployer/src/main/resources/example/instance.properties
View File

@@ -1,10 +1,10 @@
#################################################
## mysql serverId
canal.instance.mysql.slaveId=1234
canal.instance.mysql.slaveId=0
# position info
canal.instance.master.address=127.0.0.1:3306
canal.instance.master.journal.name=mysql-bin.000001
canal.instance.master.position=104606
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
@@ -27,7 +27,7 @@ canal.instance.dbPassword=canal
canal.instance.defaultDatabaseName=test
canal.instance.connectionCharset=UTF-8
# table regex
canal.instance.filter.regex=test\\..*
canal.instance.filter.regex=*\\..*
# table black regex
canal.instance.filter.black.regex=canal_tsdb\\..*
canal.instance.filter.black.regex=
#################################################

31
deployer/src/main/resources/example/rds_instance.properties Normal file
View File

@@ -0,0 +1,31 @@
#################################################
canal.instance.rds.open.url=https://rds.aliyuncs.com/
canal.instance.rds.open.accesskey=
canal.instance.rds.open.secretkey=
canal.instance.rds.instanceId=
canal.instance.rds.startTime=
canal.instance.rds.endTime=
# local binlog dir
canal.instance.parser.directory=${canal.file.data.dir:../conf}/${canal.instance.destination:}/binlog
# position info
canal.instance.master.address=127.0.0.1:3306
# table meta tsdb info
canal.instance.tsdb.enable=true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.defaultDatabaseName=test
canal.instance.connectionCharset=UTF-8
# table regex
canal.instance.filter.regex=test\\..*
# table black regex
canal.instance.filter.black.regex=
#################################################

4
deployer/src/main/resources/spring/default-instance.xml
View File

@@ -85,7 +85,7 @@
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -185,7 +185,7 @@
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
</bean>
</beans>

4
deployer/src/main/resources/spring/file-instance.xml
View File

@@ -71,7 +71,7 @@
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -170,7 +170,7 @@
<property name="supportBinlogImages" value="${canal.instance.binlog.image}" />
<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
</bean>
</beans>

4
deployer/src/main/resources/spring/group-instance.xml
View File

@@ -77,7 +77,7 @@
<bean id="eventParser1" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />
@@ -168,7 +168,7 @@
<bean id="eventParser2" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />

63
deployer/src/main/resources/spring/local-instance.xml
View File

@@ -48,41 +48,14 @@
<property name="alarmHandler">
<ref local="alarmHandler" />
</property>
<property name="eventFetcher">
<ref local="eventFetcher"/>
</property>
</bean>
<bean id="eventFetcher" class="com.alibaba.otter.canal.mq.fetch.CanalEventFetcherWithMq">
<property name="enable" value="${canal.instance.mq.enable}"/>
<property name="sendMode" value="${canal.instance.mq.sendMode}"/>
</bean>
<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.MsgAlarmHandler">
<property name="enabled" value="${canal.alarm.enable}" />
<property name="environment" value="${canal.alarm.environment}" />
<property name="intercenterApi" value="${canal.alarm.intercenter.api}" />
<property name="administrators" value="${canal.alarm.administrators}" />
</bean>
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
<bean id="zkClientx" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean" >
<property name="targetClass" value="com.alibaba.otter.canal.common.zookeeper.ZkClientx" />
<property name="targetMethod" value="getZkClient" />
<property name="arguments">
<list>
<value>${canal.zkServers:127.0.0.1:2181}</value>
</list>
</property>
</bean>
<bean id="metaManager" class="com.alibaba.otter.canal.meta.PeriodMixedMetaManager">
<property name="zooKeeperMetaManager">
<bean class="com.alibaba.otter.canal.meta.ZooKeeperMetaManager">
<property name="zkClientx" ref="zkClientx" />
</bean>
</property>
<property name="period" value="${canal.zookeeper.flush.period:1000}" />
<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
<property name="dataDir" value="${canal.file.data.dir:../conf}" />
<property name="period" value="${canal.file.flush.period:1000}" />
</bean>
<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
@@ -96,10 +69,10 @@
<property name="eventStore" ref="eventStore" />
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser">
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
@@ -144,25 +117,25 @@
</bean>
</property>
<!-- 解析起始位点 -->
<property name="masterPosition">
<bean class="com.alibaba.otter.canal.protocol.position.EntryPosition">
<property name="journalName" value="${canal.instance.master.journal.name}" />
<property name="position" value="${canal.instance.master.position}" />
<property name="timestamp" value="${canal.instance.master.timestamp}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="needWait" value="${canal.instance.parser.needWait:true}"/>
<property name="directory" value="${canal.instance.parser.directory}"/>
<property name="needWait" value="${canal.instance.parser.needWait:false}"/>
<property name="directory" value="${canal.instance.parser.directory:}"/>
<!-- rds相关 -->
<property name="url" value="${canal.instance.rds.open.url:}"/>
<property name="accesskey" value="${canal.instance.rds.open.accesskey:}"/>
<property name="secretkey" value="${canal.instance.rds.open.secretkey:}"/>
<property name="instanceId" value="${canal.instance.rds.instanceId:}"/>
<property name="startTime" value="${canal.instance.rds.startTime:}"/>
<property name="endTime" value="${canal.instance.rds.endTime:}"/>
<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:false}"/>
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
</bean>
</beans>
</beans>

2
deployer/src/main/resources/spring/memory-instance.xml
View File

@@ -68,7 +68,7 @@
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:1234}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 -->
<property name="detectingEnable" value="${canal.instance.detecting.enable:false}" />
<property name="detectingSQL" value="${canal.instance.detecting.sql}" />

2
deployer/src/main/resources/spring/tsdb/h2-tsdb.xml
View File

@@ -23,7 +23,7 @@
</bean>
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>

2
deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml
View File

@@ -23,7 +23,7 @@
</bean>
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>