chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #727 , support rds binlog dump

Browse Source
This commit is contained in:
七锋
2018-08-13 10:16:28 +08:00
committed by Chuanyi Li
parent 3053b9635a
commit 10114f4a9f
35 changed files with 553 additions and 883 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java
View File

@@ -168,7 +168,9 @@ public final class FileLogFetcher extends LogFetcher {
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#close() * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
*/ */
public void close() throws IOException { public void close() throws IOException {
if (fin != null) fin.close(); if (fin != null) {
fin.close();
}
fin = null; fin = null;
} }

5
deployer/src/main/resources/canal.properties
View File

@@ -67,6 +67,10 @@ canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MO
canal.instance.tsdb.dbUsername=canal canal.instance.tsdb.dbUsername=canal
canal.instance.tsdb.dbPassword=canal canal.instance.tsdb.dbPassword=canal
# rds oss binlog account
canal.instance.rds.accesskey =
canal.instance.rds.secretkey =
################################################# #################################################
######### destinations ############# ######### destinations #############
################################################# #################################################
@@ -83,7 +87,6 @@ canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml
canal.instance.global.mode = spring canal.instance.global.mode = spring
canal.instance.global.lazy = false canal.instance.global.lazy = false
#canal.instance.global.manager.address = 127.0.0.1:1099 #canal.instance.global.manager.address = 127.0.0.1:1099
#canal.instance.global.spring.xml = classpath:spring/local-instance.xml
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml #canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml #canal.instance.global.spring.xml = classpath:spring/default-instance.xml

5
deployer/src/main/resources/example/instance.properties
View File

@@ -12,6 +12,11 @@ canal.instance.master.position=
canal.instance.master.timestamp= canal.instance.master.timestamp=
canal.instance.master.gtid= canal.instance.master.gtid=
# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=
# table meta tsdb info # table meta tsdb info
canal.instance.tsdb.enable=true canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb

32
deployer/src/main/resources/example/rds_instance.properties
View File

@@ -1,32 +0,0 @@
#################################################
# rds openapi binlog
canal.instance.rds.open.url=https://rds.aliyuncs.com/
canal.instance.rds.open.accesskey=
canal.instance.rds.open.secretkey=
canal.instance.rds.instanceId=
canal.instance.rds.startTime=
canal.instance.rds.endTime=
# local binlog dir
canal.instance.parser.directory=${canal.file.data.dir:../conf}/${canal.instance.destination:}/binlog
# position info
canal.instance.master.address=127.0.0.1:3306
# table meta tsdb info
canal.instance.tsdb.enable=true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.defaultDatabaseName=test
canal.instance.connectionCharset=UTF-8
# table regex
canal.instance.filter.regex=test\\..*
# table black regex
canal.instance.filter.black.regex=
#################################################

39
deployer/src/main/resources/spring/base-instance.xml Normal file
View File

@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true">
<property name="accesskey" value="${canal.instance.rds.accesskey:}" />
<property name="secretkey" value="${canal.instance.rds.secretkey:}" />
<property name="instanceId" value="${canal.instance.rds.instanceId:}" />
</bean>
</beans>

23
deployer/src/main/resources/spring/default-instance.xml
View File

@@ -10,26 +10,7 @@
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd" http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName"> default-autowire="byName">
<!-- properties --> <import resource="classpath:spring/base-instance.xml" />
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring"> <bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
@@ -84,7 +65,7 @@
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/> <property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean> </bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser"> <bean id="eventParser" parent="baseEventParser" >
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" /> <property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 --> <!-- 心跳配置 -->

23
deployer/src/main/resources/spring/file-instance.xml
View File

@@ -10,26 +10,7 @@
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd" http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName"> default-autowire="byName">
<!-- properties --> <import resource="classpath:spring/base-instance.xml" />
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring"> <bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
@@ -70,7 +51,7 @@
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/> <property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean> </bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser"> <bean id="eventParser" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" /> <property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 --> <!-- 心跳配置 -->

25
deployer/src/main/resources/spring/group-instance.xml
View File

@@ -10,26 +10,7 @@
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd" http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName"> default-autowire="byName">
<!-- properties --> <import resource="classpath:spring/base-instance.xml" />
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring"> <bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
@@ -76,7 +57,7 @@
</property> </property>
</bean> </bean>
<bean id="eventParser1" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser"> <bean id="eventParser1" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" /> <property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 --> <!-- 心跳配置 -->
@@ -174,7 +155,7 @@
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" /> <property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
</bean> </bean>
<bean id="eventParser2" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser"> <bean id="eventParser2" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" /> <property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 --> <!-- 心跳配置 -->

147
deployer/src/main/resources/spring/local-instance.xml
View File

@@ -1,147 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName">
<!-- properties -->
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" />
<property name="eventParser">
<ref local="eventParser" />
</property>
<property name="eventSink">
<ref local="eventSink" />
</property>
<property name="eventStore">
<ref local="eventStore" />
</property>
<property name="metaManager">
<ref local="metaManager" />
</property>
<property name="alarmHandler">
<ref local="alarmHandler" />
</property>
</bean>
<!-- 报警处理类 -->
<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
<property name="dataDir" value="${canal.file.data.dir:../conf}" />
<property name="period" value="${canal.file.flush.period:1000}" />
</bean>
<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
</bean>
<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
<property name="eventStore" ref="eventStore" />
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser">
<property name="destination" value="${canal.instance.destination}" />
<property name="alarmHandler" ref="alarmHandler" />
<!-- 解析过滤处理 -->
<property name="eventFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
</bean>
</property>
<property name="eventBlackFilter">
<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
<constructor-arg index="1" value="false" />
</bean>
</property>
<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
<!-- 解析编码 -->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<constructor-arg ref="metaManager"/>
</bean>
</constructor-arg>
</bean>
</property>
<!-- 解析数据库信息 -->
<property name="masterInfo">
<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
<property name="address" value="${canal.instance.master.address}" />
<property name="username" value="${canal.instance.dbUsername:retl}" />
<property name="password" value="${canal.instance.dbPassword:retl}" />
<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
</bean>
</property>
<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
<property name="filterRows" value="${canal.instance.filter.rows:false}" />
<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
<property name="needWait" value="${canal.instance.parser.needWait:false}"/>
<property name="directory" value="${canal.instance.parser.directory:}"/>
<!-- rds相关 -->
<property name="url" value="${canal.instance.rds.open.url:}"/>
<property name="accesskey" value="${canal.instance.rds.open.accesskey:}"/>
<property name="secretkey" value="${canal.instance.rds.open.secretkey:}"/>
<property name="instanceId" value="${canal.instance.rds.instanceId:}"/>
<property name="startTime" value="${canal.instance.rds.startTime:}"/>
<property name="endTime" value="${canal.instance.rds.endTime:}"/>
<!--表结构相关-->
<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
<!-- parallel parser -->
<property name="parallel" value="${canal.instance.parser.parallel:true}" />
<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
</bean>
</beans>

23
deployer/src/main/resources/spring/memory-instance.xml
View File

@@ -10,26 +10,7 @@
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd" http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
default-autowire="byName"> default-autowire="byName">
<!-- properties --> <import resource="classpath:spring/base-instance.xml" />
<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
<property name="ignoreResourceNotFound" value="true" />
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
<property name="locationNames">
<list>
<value>classpath:canal.properties</value>
<value>classpath:${canal.instance.destination:}/instance.properties</value>
</list>
</property>
</bean>
<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer">
<property name="propertyEditorRegistrars">
<list>
<ref bean="socketAddressEditor" />
</list>
</property>
</bean>
<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring"> <bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
@@ -67,7 +48,7 @@
<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/> <property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
</bean> </bean>
<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser"> <bean id="eventParser" parent="baseEventParser">
<property name="destination" value="${canal.instance.destination}" /> <property name="destination" value="${canal.instance.destination}" />
<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" /> <property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
<!-- 心跳配置 --> <!-- 心跳配置 -->

2
parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java
View File

@@ -8,6 +8,8 @@ import com.alibaba.otter.canal.common.CanalException;
*/ */
public class ServerIdNotMatchException extends CanalException{ public class ServerIdNotMatchException extends CanalException{
private static final long serialVersionUID = -6124989280379293953L;
public ServerIdNotMatchException(String errorCode) { public ServerIdNotMatchException(String errorCode) {
super(errorCode); super(errorCode);
} }

8
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
View File

@@ -8,8 +8,6 @@ import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.lang.math.RandomUtils;
@@ -23,6 +21,7 @@ import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.parse.CanalEventParser; import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet; import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException; import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback; import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
@@ -99,9 +98,6 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
protected ParserExceptionHandler parserExceptionHandler; protected ParserExceptionHandler parserExceptionHandler;
protected long serverId; protected long serverId;
protected abstract BinlogParser buildParser(); protected abstract BinlogParser buildParser();
protected abstract ErosaConnection buildErosaConnection(); protected abstract ErosaConnection buildErosaConnection();
@@ -441,7 +437,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
return logPosition; return logPosition;
} }
protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, long startPosition) { protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, Long startPosition) {
if (lastPosition != null) { if (lastPosition != null) {
logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]", logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
lastPosition.getPostion()), lastPosition.getPostion()),

3
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.parse.inbound;
import com.alibaba.otter.canal.common.CanalLifeCycle; import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.taobao.tddl.dbsync.binlog.LogBuffer; import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/** /**
* 针对解析器提供一个多阶段协同的处理 * 针对解析器提供一个多阶段协同的处理
@@ -23,7 +24,7 @@ public interface MultiStageCoprocessor extends CanalLifeCycle {
*/ */
public boolean publish(LogBuffer buffer); public boolean publish(LogBuffer buffer);
public boolean publish(LogBuffer buffer, String binlogFileName); public boolean publish(LogEvent event);
public void reset(); public void reset();
} }

39
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
View File

@@ -4,25 +4,24 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet; import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
import com.alibaba.otter.canal.parse.inbound.ErosaConnection; import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor; import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
import com.alibaba.otter.canal.parse.inbound.SinkFunction; import com.alibaba.otter.canal.parse.inbound.SinkFunction;
import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue; import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue;
import com.taobao.tddl.dbsync.binlog.FileLogFetcher; import com.taobao.tddl.dbsync.binlog.FileLogFetcher;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent; import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.LogPosition; import com.taobao.tddl.dbsync.binlog.LogPosition;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
/** /**
* local bin log connection (not real connection) * local bin log connection (not real connection)
@@ -40,7 +39,6 @@ public class LocalBinLogConnection implements ErosaConnection {
private long serverId; private long serverId;
private FileParserListener parserListener; private FileParserListener parserListener;
public LocalBinLogConnection(){ public LocalBinLogConnection(){
} }
@@ -92,9 +90,6 @@ public class LocalBinLogConnection implements ErosaConnection {
while (running) { while (running) {
boolean needContinue = true; boolean needContinue = true;
LogEvent event = null; LogEvent event = null;
// 处理一下binlog文件名
event = new RotateLogEvent(context.getLogPosition().getFileName(), 4);
func.sink(event);
while (fetcher.fetch()) { while (fetcher.fetch()) {
event = decoder.decode(fetcher, context); event = decoder.decode(fetcher, context);
if (event == null) { if (event == null) {
@@ -168,7 +163,6 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) { while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context); LogEvent event = decoder.decode(fetcher, context);
if (event != null) { if (event != null) {
if (serverId != 0 && event.getServerId() != serverId) { if (serverId != 0 && event.getServerId() != serverId) {
throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !"); throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
} }
@@ -189,6 +183,9 @@ public class LocalBinLogConnection implements ErosaConnection {
} else if (LogEvent.XID_EVENT == event.getHeader().getType()) { } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
lastXidLogFilename = current.getName(); lastXidLogFilename = current.getName();
lastXidLogFileOffset = event.getLogPos(); lastXidLogFileOffset = event.getLogPos();
} else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
lastXidLogFilename = current.getName();
lastXidLogFileOffset = event.getLogPos();
} }
} }
} }
@@ -225,17 +222,29 @@ public class LocalBinLogConnection implements ErosaConnection {
@Override @Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException { public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
File current = new File(directory, binlogfilename); File current = new File(directory, binlogfilename);
if (!current.exists()) {
throw new CanalParseException("binlog:" + binlogfilename + " is not found");
}
FileLogFetcher fetcher = new FileLogFetcher(bufferSize); FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
LogContext context = new LogContext();
try { try {
fetcher.open(current, binlogPosition); fetcher.open(current, binlogPosition);
context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
while (running) { while (running) {
boolean needContinue = true; boolean needContinue = true;
LogEvent event = null;
while (fetcher.fetch()) { while (fetcher.fetch()) {
LogBuffer buffer = fetcher.duplicate(); event = decoder.decode(fetcher, context);
fetcher.consume(fetcher.limit()); if (event == null) {
// set filename continue;
if (!coprocessor.publish(buffer, binlogfilename)) { }
if (serverId != 0 && event.getServerId() != serverId) {
throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
}
if (!coprocessor.publish(event)) {
needContinue = false; needContinue = false;
break; break;
} }
@@ -244,7 +253,6 @@ public class LocalBinLogConnection implements ErosaConnection {
fetcher.close(); // 关闭上一个文件 fetcher.close(); // 关闭上一个文件
parserFinish(binlogfilename); parserFinish(binlogfilename);
if (needContinue) {// 读取下一个 if (needContinue) {// 读取下一个
File nextFile; File nextFile;
if (needWait) { if (needWait) {
nextFile = binlogs.waitForNextFile(current); nextFile = binlogs.waitForNextFile(current);
@@ -306,7 +314,6 @@ public class LocalBinLogConnection implements ErosaConnection {
while (fetcher.fetch()) { while (fetcher.fetch()) {
LogEvent event = decoder.decode(fetcher, context); LogEvent event = decoder.decode(fetcher, context);
if (event != null) { if (event != null) {
if (serverId != 0 && event.getServerId() != serverId) { if (serverId != 0 && event.getServerId() != serverId) {
throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !"); throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
} }
@@ -327,6 +334,9 @@ public class LocalBinLogConnection implements ErosaConnection {
} else if (LogEvent.XID_EVENT == event.getHeader().getType()) { } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
lastXidLogFilename = current.getName(); lastXidLogFilename = current.getName();
lastXidLogFileOffset = event.getLogPos(); lastXidLogFileOffset = event.getLogPos();
} else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
lastXidLogFilename = current.getName();
lastXidLogFileOffset = event.getLogPos();
} }
} }
} }
@@ -411,6 +421,7 @@ public class LocalBinLogConnection implements ErosaConnection {
} }
public interface FileParserListener { public interface FileParserListener {
void onFinish(String fileName); void onFinish(String fileName);
} }

3
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -13,7 +13,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils; import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector; import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor; import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
@@ -478,8 +477,6 @@ public class MysqlConnection implements ErosaConnection {
} }
} }
public static enum BinlogFormat { public static enum BinlogFormat {
STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED"); STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED");

11
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -756,17 +756,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
Long logposTimestamp = entry.getHeader().getExecuteTime(); Long logposTimestamp = entry.getHeader().getExecuteTime();
Long serverId = entry.getHeader().getServerId(); Long serverId = entry.getHeader().getServerId();
if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) // 如果最小的一条记录都不满足条件，可直接退出
|| CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
if (logger.isDebugEnabled()) {
logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] {
logfilename, logfileoffset, logposTimestamp, startTimestamp });
}
// 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
if (logposTimestamp >= startTimestamp) { if (logposTimestamp >= startTimestamp) {
return false; return false;
} }
}
if (StringUtils.equals(endPosition.getJournalName(), logfilename) if (StringUtils.equals(endPosition.getJournalName(), logfilename)
&& endPosition.getPosition() <= logfileoffset) { && endPosition.getPosition() <= logfileoffset) {
@@ -914,8 +907,6 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
this.dumpErrorCountThreshold = dumpErrorCountThreshold; this.dumpErrorCountThreshold = dumpErrorCountThreshold;
} }
public AtomicLong getReceivedBinlogBytes() { public AtomicLong getReceivedBinlogBytes() {
return this.receivedBinlogBytes; return this.receivedBinlogBytes;
} }

51
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
View File

@@ -5,8 +5,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.LockSupport;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.exception.CanalParseException;
@@ -32,7 +30,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogContext;
import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogDecoder;
import com.taobao.tddl.dbsync.binlog.LogEvent; import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.LogPosition;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
@@ -141,14 +138,18 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
super.stop(); super.stop();
} }
public boolean publish(LogBuffer buffer) {
return this.publish(buffer, null);
}
/** /**
* 网络数据投递 * 网络数据投递
*/ */
public boolean publish(LogBuffer buffer) { public boolean publish(LogEvent event) {
return publish(buffer, null); return this.publish(null, event);
} }
public boolean publish(LogBuffer buffer, String binlogFileName) { private boolean publish(LogBuffer buffer, LogEvent event) {
if (!isStart()) { if (!isStart()) {
if (exception != null) { if (exception != null) {
throw exception; throw exception;
@@ -169,10 +170,11 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
do { do {
try { try {
long next = disruptorMsgBuffer.tryNext(); long next = disruptorMsgBuffer.tryNext();
MessageEvent event = disruptorMsgBuffer.get(next); MessageEvent data = disruptorMsgBuffer.get(next);
event.setBuffer(buffer); if (buffer != null) {
if (binlogFileName != null) { data.setBuffer(buffer);
event.setBinlogFileName(binlogFileName); } else {
data.setEvent(event);
} }
disruptorMsgBuffer.publish(next); disruptorMsgBuffer.publish(next);
if (fullTimes > 0) { if (fullTimes > 0) {
@@ -217,7 +219,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
start(); start();
} }
private class SimpleParserStage implements EventHandler<MessageEvent>, LifecycleAware { private class SimpleParserStage implements EventHandler<MessageEvent>, LifecycleAware {
private LogDecoder decoder; private LogDecoder decoder;
@@ -230,22 +231,12 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception { public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
try { try {
LogEvent logEvent = event.getEvent();
if (logEvent == null) {
LogBuffer buffer = event.getBuffer(); LogBuffer buffer = event.getBuffer();
if (StringUtils.isNotEmpty(event.getBinlogFileName()) logEvent = decoder.decode(buffer, context);
&& (context.getLogPosition() == null || !context.getLogPosition()
.getFileName()
.equals(event.getBinlogFileName()))) {
// set roate binlog file name
if (context.getLogPosition() == null) {
context.setLogPosition(new LogPosition(event.getBinlogFileName(), 0));
} else {
context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
.getPosition()));
}
}
LogEvent logEvent = decoder.decode(buffer, context);
event.setEvent(logEvent); event.setEvent(logEvent);
}
int eventType = logEvent.getHeader().getType(); int eventType = logEvent.getHeader().getType();
TableMeta tableMeta = null; TableMeta tableMeta = null;
@@ -347,7 +338,6 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
// clear for gc // clear for gc
event.setBuffer(null); event.setBuffer(null);
event.setBinlogFileName(null);
event.setEvent(null); event.setEvent(null);
event.setTable(null); event.setTable(null);
event.setEntry(null); event.setEntry(null);
@@ -371,21 +361,12 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
class MessageEvent { class MessageEvent {
private String binlogFileName; // for local binlog parse
private LogBuffer buffer; private LogBuffer buffer;
private CanalEntry.Entry entry; private CanalEntry.Entry entry;
private boolean needDmlParse = false; private boolean needDmlParse = false;
private TableMeta table; private TableMeta table;
private LogEvent event; private LogEvent event;
public String getBinlogFileName() {
return binlogFileName;
}
public void setBinlogFileName(String binlogFileName) {
this.binlogFileName = binlogFileName;
}
public LogBuffer getBuffer() { public LogBuffer getBuffer() {
return buffer; return buffer;
} }

22
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -10,7 +10,6 @@ import java.util.Arrays;
import java.util.BitSet; import java.util.BitSet;
import java.util.List; import java.util.List;
import com.taobao.tddl.dbsync.binlog.event.*;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -41,7 +40,23 @@ import com.alibaba.otter.canal.protocol.CanalEntry.Type;
import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.taobao.tddl.dbsync.binlog.LogEvent; import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo; import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
/** /**
@@ -872,11 +887,14 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) { private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
try { try {
return tableMetaCache.getTableMeta(dbName, tbName, useCache, position); return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
} catch (Exception e) { } catch (Throwable e) {
String message = ExceptionUtils.getRootCauseMessage(e); String message = ExceptionUtils.getRootCauseMessage(e);
if (filterTableError) { if (filterTableError) {
if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) { if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
return null; return null;
} else if (StringUtils.contains(message, "errorNumber=1142")
&& StringUtils.contains(message, "command denied")) {
return null;
} }
} }

34
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java
View File

@@ -32,6 +32,7 @@ public class BinLogFileQueue {
private Condition nextCondition = lock.newCondition(); private Condition nextCondition = lock.newCondition();
private Timer timer = new Timer(true); private Timer timer = new Timer(true);
private long reloadInterval = 10 * 1000L; // 10秒 private long reloadInterval = 10 * 1000L; // 10秒
private CanalParseException exception = null;
public BinLogFileQueue(String directory){ public BinLogFileQueue(String directory){
this(new File(directory)); this(new File(directory));
@@ -52,10 +53,25 @@ public class BinLogFileQueue {
timer.scheduleAtFixedRate(new TimerTask() { timer.scheduleAtFixedRate(new TimerTask() {
public void run() { public void run() {
try {
// File errorFile = new File(BinLogFileQueue.this.directory,
// errorFileName);
// if (errorFile.isFile() && errorFile.exists()) {
// String text = StringUtils.join(IOUtils.readLines(new
// FileInputStream(errorFile)), "\n");
// exception = new CanalParseException(text);
// }
List<File> files = listBinlogFiles(); List<File> files = listBinlogFiles();
for (File file : files) { for (File file : files) {
offer(file); offer(file);
} }
} catch (Throwable e) {
exception = new CanalParseException(e);
}
if (exception != null) {
offer(null);
}
} }
}, reloadInterval, reloadInterval); }, reloadInterval, reloadInterval);
} }
@@ -69,6 +85,10 @@ public class BinLogFileQueue {
public File getNextFile(File pre) { public File getNextFile(File pre) {
try { try {
lock.lockInterruptibly(); lock.lockInterruptibly();
if (exception != null) {
throw exception;
}
if (binlogs.size() == 0) { if (binlogs.size() == 0) {
return null; return null;
} else { } else {
@@ -94,6 +114,10 @@ public class BinLogFileQueue {
public File getBefore(File file) { public File getBefore(File file) {
try { try {
lock.lockInterruptibly(); lock.lockInterruptibly();
if (exception != null) {
throw exception;
}
if (binlogs.size() == 0) { if (binlogs.size() == 0) {
return null; return null;
} else { } else {
@@ -130,6 +154,9 @@ public class BinLogFileQueue {
nextCondition.await();// 等待新文件 nextCondition.await();// 等待新文件
} }
if (exception != null) {
throw exception;
}
if (pre == null) {// 第一次 if (pre == null) {// 第一次
return binlogs.get(0); return binlogs.get(0);
} else { } else {
@@ -170,13 +197,16 @@ public class BinLogFileQueue {
private boolean offer(File file) { private boolean offer(File file) {
try { try {
lock.lockInterruptibly(); lock.lockInterruptibly();
if (file != null) {
if (!binlogs.contains(file)) { if (!binlogs.contains(file)) {
binlogs.add(file); binlogs.add(file);
nextCondition.signalAll();// 唤醒 nextCondition.signalAll();// 唤醒
return true; return true;
} else {
return false;
} }
}
nextCondition.signalAll();// 唤醒
return false;
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
return false; return false;

97
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
View File

@@ -7,6 +7,8 @@ import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
@@ -14,6 +16,9 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;
import javax.net.ssl.SSLContext;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -23,8 +28,16 @@ import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -42,7 +55,7 @@ public class BinlogDownloadQueue {
private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<BinlogFile>(); private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<BinlogFile>();
private LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<Runnable>(); private LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<Runnable>();
private LinkedList<BinlogFile> binlogList; private LinkedList<BinlogFile> binlogList;
private final int batchSize; private final int batchFileSize;
private Thread downloadThread; private Thread downloadThread;
public boolean running = true; public boolean running = true;
private final String destDir; private final String destDir;
@@ -50,9 +63,9 @@ public class BinlogDownloadQueue {
private int currentSize; private int currentSize;
private String lastDownload; private String lastDownload;
public BinlogDownloadQueue(List<BinlogFile> downloadQueue, int batchSize, String destDir) throws IOException{ public BinlogDownloadQueue(List<BinlogFile> downloadQueue, int batchFileSize, String destDir) throws IOException{
this.binlogList = new LinkedList(downloadQueue); this.binlogList = new LinkedList(downloadQueue);
this.batchSize = batchSize; this.batchFileSize = batchFileSize;
this.destDir = destDir; this.destDir = destDir;
this.currentSize = 0; this.currentSize = 0;
prepareBinlogList(); prepareBinlogList();
@@ -83,11 +96,12 @@ public class BinlogDownloadQueue {
if (downloadThread != null) { if (downloadThread != null) {
return; return;
} }
downloadThread = new Thread(new DownloadThread()); downloadThread = new Thread(new DownloadThread(), "download-" + destDir);
downloadThread.setDaemon(true);
downloadThread.start(); downloadThread.start();
} }
public BinlogFile tryOne() throws IOException { public BinlogFile tryOne() throws Throwable {
BinlogFile binlogFile = binlogList.poll(); BinlogFile binlogFile = binlogList.poll();
download(binlogFile); download(binlogFile);
hostId = binlogFile.getHostInstanceID(); hostId = binlogFile.getHostInstanceID();
@@ -121,7 +135,7 @@ public class BinlogDownloadQueue {
} }
public void prepare() throws InterruptedException { public void prepare() throws InterruptedException {
for (int i = this.currentSize; i < batchSize && !binlogList.isEmpty(); i++) { for (int i = this.currentSize; i < batchFileSize && !binlogList.isEmpty(); i++) {
BinlogFile binlogFile = null; BinlogFile binlogFile = null;
while (!binlogList.isEmpty()) { while (!binlogList.isEmpty()) {
binlogFile = binlogList.poll(); binlogFile = binlogList.poll();
@@ -150,11 +164,37 @@ public class BinlogDownloadQueue {
downloadQueue.clear(); downloadQueue.clear();
} }
private void download(BinlogFile binlogFile) throws IOException { private void download(BinlogFile binlogFile) throws Throwable {
String downloadLink = binlogFile.getDownloadLink(); String downloadLink = binlogFile.getDownloadLink();
String fileName = binlogFile.getFileName(); String fileName = binlogFile.getFileName();
downloadLink = downloadLink.trim();
CloseableHttpClient httpClient = null;
if (downloadLink.startsWith("https")) {
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
// 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext)
.setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
.build()))
.build();
} else {
httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
}
HttpGet httpGet = new HttpGet(downloadLink); HttpGet httpGet = new HttpGet(downloadLink);
CloseableHttpClient httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
RequestConfig requestConfig = RequestConfig.custom() RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(TIMEOUT) .setConnectTimeout(TIMEOUT)
.setConnectionRequestTimeout(TIMEOUT) .setConnectionRequestTimeout(TIMEOUT)
@@ -187,6 +227,9 @@ public class BinlogDownloadQueue {
String name = tarArchiveEntry.getName(); String name = tarArchiveEntry.getName();
File tarFile = new File(parentFile, name + ".tmp"); File tarFile = new File(parentFile, name + ".tmp");
logger.info("start to download file " + tarFile.getName()); logger.info("start to download file " + tarFile.getName());
if (tarFile.exists()) {
tarFile.delete();
}
BufferedOutputStream bos = null; BufferedOutputStream bos = null;
try { try {
bos = new BufferedOutputStream(new FileOutputStream(tarFile)); bos = new BufferedOutputStream(new FileOutputStream(tarFile));
@@ -204,6 +247,10 @@ public class BinlogDownloadQueue {
tais.close(); tais.close();
} else { } else {
File file = new File(parentFile, fileName + ".tmp"); File file = new File(parentFile, fileName + ".tmp");
if (file.exists()) {
file.delete();
}
if (!file.isFile()) { if (!file.isFile()) {
file.createNewFile(); file.createNewFile();
} }
@@ -245,17 +292,45 @@ public class BinlogDownloadQueue {
@Override @Override
public void run() { public void run() {
while (running) { while (running) {
BinlogFile binlogFile = null;
try { try {
BinlogFile binlogFile = downloadQueue.poll(5000, TimeUnit.MILLISECONDS); binlogFile = downloadQueue.poll(5000, TimeUnit.MILLISECONDS);
if (binlogFile != null) { if (binlogFile != null) {
int retry = 1;
while (true) {
try {
download(binlogFile); download(binlogFile);
break;
} catch (Throwable e) {
if (retry % 10 == 0) {
retry = retry + 1;
try {
logger.warn("download failed + " + binlogFile.toString() + "], retry : "
+ retry, e);
// File errorFile = new File(destDir,
// "error.txt");
// FileWriter writer = new
// FileWriter(errorFile);
// writer.write(ExceptionUtils.getFullStackTrace(e));
// writer.flush();
// IOUtils.closeQuietly(writer);
LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100 * retry));
} catch (Throwable e1) {
logger.error("write error failed", e1);
} }
} else {
retry = retry + 1;
}
}
}
}
Runnable runnable = taskQueue.poll(5000, TimeUnit.MILLISECONDS); Runnable runnable = taskQueue.poll(5000, TimeUnit.MILLISECONDS);
if (runnable != null) { if (runnable != null) {
runnable.run(); runnable.run();
} }
} catch (Exception e) { } catch (Throwable e) {
e.printStackTrace(); logger.error("task process failed", e);
} }
} }

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java
View File

@@ -155,6 +155,7 @@ public class HttpHelper {
.register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)) .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
.build())) .build()))
.build(); .build();
// ---------------- 创建支持https 的client成功--------- // ---------------- 创建支持https 的client成功---------
URI uri = new URIBuilder(url).build(); URI uri = new URIBuilder(url).build();

97
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
View File

@@ -4,11 +4,19 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadFactory;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.parse.exception.PositionNotFoundException; import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler; import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
/** /**
* aliyun rds的binlog parser支持
*
* <pre>
* 注意点：aliyun的binlog会有定期清理并备份到oss上, 这里实现了一份自动下载oss+rds binlog的机制
* </pre>
*
* @author chengjin.lyf on 2018/7/20 上午10:52 * @author chengjin.lyf on 2018/7/20 上午10:52
* @since 1.0.25 * @since 1.0.25
*/ */
@@ -18,18 +26,16 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
private String accesskey; // 云账号的ak private String accesskey; // 云账号的ak
private String secretkey; // 云账号sk private String secretkey; // 云账号sk
private String instanceId; // rds实例id private String instanceId; // rds实例id
private Long startTime; private String directory; // binlog目录
private Long endTime; private int batchFileSize = 4; // 最多下载的binlog文件数量
private String directory; // binlog
// 目录
private int batchSize = 4; // 最多下载的binlog文件数量
private RdsLocalBinlogEventParser rdsBinlogEventParser = new RdsLocalBinlogEventParser(); private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser();
private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() { private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() {
@Override @Override
public Thread newThread(Runnable r) { public Thread newThread(Runnable r) {
Thread t = new Thread(r, "rds-binlog-daemon-thread"); Thread t = new Thread(r,
"rds-binlog-daemon-thread");
t.setDaemon(true); t.setDaemon(true);
return t; return t;
} }
@@ -37,32 +43,40 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
@Override @Override
public void start() { public void start() {
if (StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey)
&& StringUtils.isNotEmpty(instanceId)) {
final ParserExceptionHandler targetHandler = this.getParserExceptionHandler(); final ParserExceptionHandler targetHandler = this.getParserExceptionHandler();
rdsBinlogEventParser.setLogPositionManager(this.getLogPositionManager()); if (directory == null) {
rdsBinlogEventParser.setDestination(destination); directory = System.getProperty("java.io.tmpdir", "/tmp") + "/" + destination;
rdsBinlogEventParser.setAlarmHandler(this.getAlarmHandler()); }
rdsBinlogEventParser.setConnectionCharset(this.connectionCharset); rdsLocalBinlogEventParser.setLogPositionManager(this.getLogPositionManager());
rdsBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber); rdsLocalBinlogEventParser.setDestination(destination);
rdsBinlogEventParser.setEnableTsdb(this.enableTsdb); rdsLocalBinlogEventParser.setAlarmHandler(this.getAlarmHandler());
rdsBinlogEventParser.setEventBlackFilter(this.eventBlackFilter); rdsLocalBinlogEventParser.setConnectionCharset(this.connectionCharset);
rdsBinlogEventParser.setFilterQueryDcl(this.filterQueryDcl); rdsLocalBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber);
rdsBinlogEventParser.setFilterQueryDdl(this.filterQueryDdl); rdsLocalBinlogEventParser.setEnableTsdb(this.enableTsdb);
rdsBinlogEventParser.setFilterQueryDml(this.filterQueryDml); rdsLocalBinlogEventParser.setEventBlackFilter(this.eventBlackFilter);
rdsBinlogEventParser.setFilterRows(this.filterRows); rdsLocalBinlogEventParser.setFilterQueryDcl(this.filterQueryDcl);
rdsBinlogEventParser.setFilterTableError(this.filterTableError); rdsLocalBinlogEventParser.setFilterQueryDdl(this.filterQueryDdl);
rdsBinlogEventParser.setIsGTIDMode(this.isGTIDMode); rdsLocalBinlogEventParser.setFilterQueryDml(this.filterQueryDml);
rdsBinlogEventParser.setMasterInfo(this.masterInfo); rdsLocalBinlogEventParser.setFilterRows(this.filterRows);
rdsBinlogEventParser.setEventFilter(this.eventFilter); rdsLocalBinlogEventParser.setFilterTableError(this.filterTableError);
rdsBinlogEventParser.setMasterPosition(this.masterPosition); // rdsLocalBinlogEventParser.setIsGTIDMode(this.isGTIDMode);
rdsBinlogEventParser.setTransactionSize(this.transactionSize); rdsLocalBinlogEventParser.setMasterInfo(this.masterInfo);
rdsBinlogEventParser.setUrl(this.rdsOpenApiUrl); rdsLocalBinlogEventParser.setEventFilter(this.eventFilter);
rdsBinlogEventParser.setAccesskey(this.accesskey); rdsLocalBinlogEventParser.setMasterPosition(this.masterPosition);
rdsBinlogEventParser.setSecretkey(this.secretkey); rdsLocalBinlogEventParser.setTransactionSize(this.transactionSize);
rdsBinlogEventParser.setInstanceId(this.instanceId); rdsLocalBinlogEventParser.setUrl(this.rdsOpenApiUrl);
rdsBinlogEventParser.setEventSink(eventSink); rdsLocalBinlogEventParser.setAccesskey(this.accesskey);
rdsBinlogEventParser.setDirectory(directory); rdsLocalBinlogEventParser.setSecretkey(this.secretkey);
rdsBinlogEventParser.setBatchSize(batchSize); rdsLocalBinlogEventParser.setInstanceId(this.instanceId);
rdsBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() { rdsLocalBinlogEventParser.setEventSink(eventSink);
rdsLocalBinlogEventParser.setDirectory(directory);
rdsLocalBinlogEventParser.setBatchFileSize(batchFileSize);
rdsLocalBinlogEventParser.setParallel(this.parallel);
rdsLocalBinlogEventParser.setParallelBufferSize(this.parallelBufferSize);
rdsLocalBinlogEventParser.setParallelThreadSize(this.parallelThreadSize);
rdsLocalBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() {
@Override @Override
public void onFinish() { public void onFinish() {
@@ -70,7 +84,7 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
@Override @Override
public void run() { public void run() {
rdsBinlogEventParser.stop(); rdsLocalBinlogEventParser.stop();
RdsBinlogEventParserProxy.this.start(); RdsBinlogEventParserProxy.this.start();
} }
}); });
@@ -87,10 +101,12 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
} }
} }
}); });
}
super.start(); super.start();
} }
public void handleMysqlParserException(Throwable throwable) { private void handleMysqlParserException(Throwable throwable) {
if (throwable instanceof PositionNotFoundException) { if (throwable instanceof PositionNotFoundException) {
logger.info("remove rds not found position, try download rds binlog!"); logger.info("remove rds not found position, try download rds binlog!");
executorService.execute(new Runnable() { executorService.execute(new Runnable() {
@@ -101,12 +117,12 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
logger.info("stop mysql parser!"); logger.info("stop mysql parser!");
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this; RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
long serverId = rdsBinlogEventParserProxy.getServerId(); long serverId = rdsBinlogEventParserProxy.getServerId();
rdsBinlogEventParser.setServerId(serverId); rdsLocalBinlogEventParser.setServerId(serverId);
rdsBinlogEventParserProxy.stop(); rdsBinlogEventParserProxy.stop();
logger.info("start rds mysql binlog parser!"); logger.info("start rds mysql binlog parser!");
rdsBinlogEventParser.start(); rdsLocalBinlogEventParser.start();
} catch (Exception e) { } catch (Throwable e) {
e.printStackTrace(); logger.info("handle exception failed", e);
} }
} }
}); });
@@ -143,7 +159,8 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
this.directory = directory; this.directory = directory;
} }
public void setBatchSize(int batchSize) { public void setBatchFileSize(int batchFileSize) {
this.batchSize = batchSize; this.batchFileSize = batchFileSize;
} }
} }

330
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java
View File

@@ -1,51 +1,20 @@
package com.alibaba.otter.canal.parse.inbound.mysql.rds; package com.alibaba.otter.canal.parse.inbound.mysql.rds;
import io.netty.handler.codec.http.HttpResponseStatus;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.TreeMap;
import java.util.UUID;
import javax.crypto.Mac;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile; import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult; import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsItem; import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsItem;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBackupPolicyRequest;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFilesRequest; import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFilesRequest;
/** /**
@@ -55,12 +24,6 @@ import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFil
public class RdsBinlogOpenApi { public class RdsBinlogOpenApi {
protected static final Logger logger = LoggerFactory.getLogger(RdsBinlogOpenApi.class); protected static final Logger logger = LoggerFactory.getLogger(RdsBinlogOpenApi.class);
private static final String ISO8601_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
private static final int TIMEOUT = 10000;
private static final String ENCODING = "UTF-8";
private static final String MAC_NAME = "HmacSHA1";
private static final String API_VERSION = "2014-08-15";
private static final String SIGNATURE_VERSION = "1.0";
public static List<BinlogFile> listBinlogFiles(String url, String ak, String sk, String dbInstanceId, public static List<BinlogFile> listBinlogFiles(String url, String ak, String sk, String dbInstanceId,
Date startTime, Date endTime) { Date startTime, Date endTime) {
@@ -106,279 +69,32 @@ public class RdsBinlogOpenApi {
return Collections.EMPTY_LIST; return Collections.EMPTY_LIST;
} }
public static void downloadBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime, public static RdsBackupPolicy queryBinlogBackupPolicy(String url, String ak, String sk, String dbInstanceId) {
Date endTime, File destDir) throws Throwable { DescribeBackupPolicyRequest request = new DescribeBackupPolicyRequest();
int pageSize = 100; if (StringUtils.isNotEmpty(url)) {
int pageNumber = 0;
int pageRecordCount = 1;
String hostInstanceID = null;
while (pageRecordCount > 0 && pageRecordCount <= pageSize) {
pageNumber += 1;
String result = describeBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime, pageSize, pageNumber);
JSONObject jsobObj = JSON.parseObject(result);
pageRecordCount = jsobObj.getInteger("PageRecordCount");
if (pageRecordCount > 0) {
FileUtils.forceMkdir(destDir);
File hostIdFile = new File(destDir, "hostId");
if (hostIdFile.exists()) {
List<String> lines = IOUtils.readLines(new FileInputStream(hostIdFile));
hostInstanceID = StringUtils.join(lines, "\n");
}
String itemStr = jsobObj.getString("Items");
JSONObject binLogFileObj = JSONObject.parseObject(itemStr);
JSONArray items = binLogFileObj.getJSONArray("BinLogFile");
if (items == null || items.isEmpty()) {
continue;
}
for (int i = 0; i < items.size(); i++) {
JSONObject item = (JSONObject) items.get(i);
String oneHostInstanceID = item.getString("HostInstanceID");
if (hostInstanceID == null) {
hostInstanceID = oneHostInstanceID;
FileOutputStream hostIdFileOut = null;
try { try {
hostIdFileOut = new FileOutputStream(hostIdFile); URI uri = new URI(url);
hostIdFileOut.write(oneHostInstanceID.getBytes()); request.setEndPoint(uri.getHost());
hostIdFileOut.flush(); } catch (URISyntaxException e) {
} finally { logger.error("resolve url host failed, will use default rds endpoint!");
IOUtils.closeQuietly(hostIdFileOut);
} }
} }
request.setRdsInstanceId(dbInstanceId);
if (hostInstanceID.equals(oneHostInstanceID)) { // 只选择一个host下载 request.setAccessKeyId(ak);
String downloadLink = item.getString("DownloadLink"); request.setAccessKeySecret(sk);
String fileName = StringUtils.substringBetween(downloadLink, "mysql-bin.", ".tar"); int retryTime = 3;
if (StringUtils.isNotEmpty(fileName)) { while (true) {
File currentFile = new File(destDir, "mysql-bin." + fileName);
if (currentFile.isFile() && currentFile.exists()) {
// 检查一下文件是否存在，存在就就没必要下载了
continue;
}
}
HttpGet httpGet = new HttpGet(downloadLink);
CloseableHttpClient httpClient = HttpClientBuilder.create()
.setMaxConnPerRoute(50)
.setMaxConnTotal(100)
.build();
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(TIMEOUT)
.setConnectionRequestTimeout(TIMEOUT)
.setSocketTimeout(TIMEOUT)
.build();
httpGet.setConfig(requestConfig);
HttpResponse response = httpClient.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpResponseStatus.OK.code()) {
throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:"
+ statusCode);
}
saveFile(destDir, response);
}
}
}
}
}
private static void saveFile(File parentFile, HttpResponse response) throws IOException {
InputStream is = response.getEntity().getContent();
long totalSize = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
String fileName = response.getFirstHeader("Content-Disposition").getValue();
fileName = StringUtils.substringAfter(fileName, "filename=");
boolean isTar = StringUtils.endsWith(fileName, ".tar");
FileUtils.forceMkdir(parentFile);
FileOutputStream fos = null;
try { try {
if (isTar) { return request.doAction();
TarArchiveInputStream tais = new TarArchiveInputStream(is);
TarArchiveEntry tarArchiveEntry = null;
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
String name = tarArchiveEntry.getName();
File tarFile = new File(parentFile, name);
logger.info("start to download file " + tarFile.getName());
BufferedOutputStream bos = null;
try {
bos = new BufferedOutputStream(new FileOutputStream(tarFile));
int read = -1;
byte[] buffer = new byte[1024];
while ((read = tais.read(buffer)) != -1) {
bos.write(buffer, 0, read);
}
logger.info("download file " + tarFile.getName() + " end!");
} finally {
IOUtils.closeQuietly(bos);
}
}
tais.close();
} else {
File file = new File(parentFile, fileName);
if (!file.isFile()) {
file.createNewFile();
}
try {
fos = new FileOutputStream(file);
byte[] buffer = new byte[1024];
int len;
long copySize = 0;
long nextPrintProgress = 0;
logger.info("start to download file " + file.getName());
while ((len = is.read(buffer)) != -1) {
fos.write(buffer, 0, len);
copySize += len;
long progress = copySize * 100 / totalSize;
if (progress >= nextPrintProgress) {
logger.info("download " + file.getName() + " progress : " + progress
+ "% , download size : " + copySize + ", total size : " + totalSize);
nextPrintProgress += 10;
}
}
logger.info("download file " + file.getName() + " end!");
fos.flush();
} finally {
IOUtils.closeQuietly(fos);
}
}
} finally {
IOUtils.closeQuietly(fos);
}
}
public static String describeBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
Date endTime, int pageSize, int pageNumber) throws Exception {
Map<String, String> paramMap = new HashMap<String, String>();
paramMap.put("Action", "DescribeBinlogFiles");
paramMap.put("DBInstanceId", dbInstanceId); // rds实例id
paramMap.put("StartTime", formatIso8601Date(startTime));
paramMap.put("EndTime", formatIso8601Date(endTime));
paramMap.put("PageSize", String.valueOf(pageSize));
paramMap.put("PageNumber", String.valueOf(pageNumber));
return doRequest(url, paramMap, ak, sk);
}
private static String doRequest(String domin, Map<String, String> param, String ak, String sk) throws Exception {
param.put("AccessKeyId", ak);
param.put("SignatureMethod", "HMAC-SHA1");
param.put("SignatureVersion", SIGNATURE_VERSION);
param.put("Version", API_VERSION);
param.put("SignatureNonce", UUID.randomUUID().toString());
param.put("Format", "JSON");
param.put("Timestamp", formatIso8601Date(new Date()));
String signStr = generate("POST", param, sk);
param.put("Signature", signStr);
String request = concatQueryString(param);
String url = domin + "?" + request;
String result = HttpHelper.post(url, null, Collections.EMPTY_MAP, TIMEOUT);
return result;
}
public static String concatQueryString(Map<String, String> parameters) throws UnsupportedEncodingException {
if (null == parameters) {
return null;
}
StringBuilder urlBuilder = new StringBuilder("");
for (Map.Entry<String, String> entry : parameters.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
urlBuilder.append(encode(key));
if (val != null) {
urlBuilder.append("=").append(encode(val));
}
urlBuilder.append("&");
}
int strIndex = urlBuilder.length();
if (parameters.size() > 0) {
urlBuilder.deleteCharAt(strIndex - 1);
}
return urlBuilder.toString();
}
public static String encode(String value) throws UnsupportedEncodingException {
return URLEncoder.encode(value, "UTF-8");
}
private static String formatIso8601Date(Date date) {
SimpleDateFormat df = new SimpleDateFormat(ISO8601_DATE_FORMAT);
df.setTimeZone(TimeZone.getTimeZone("GMT"));
return df.format(date);
}
/**
* 使用 HMAC-SHA1 签名方法对对encryptText进行签名
*
* @param encryptText 被签名的字符串
* @param encryptKey 密钥
* @return
* @throws Exception
*/
public static byte[] HmacSHA1Encrypt(String encryptText, String encryptKey) throws Exception {
byte[] data = encryptKey.getBytes(ENCODING);
// 根据给定的字节数组构造一个密钥,第二参数指定一个密钥算法的名称
SecretKey secretKey = new SecretKeySpec(data, MAC_NAME);
// 生成一个指定 Mac 算法 的 Mac 对象
Mac mac = Mac.getInstance(MAC_NAME);
// 用给定密钥初始化 Mac 对象
mac.init(secretKey);
byte[] text = encryptText.getBytes(ENCODING);
// 完成 Mac 操作
return mac.doFinal(text);
}
private static String base64(byte input[]) throws UnsupportedEncodingException {
return new String(Base64.encodeBase64(input), ENCODING);
}
/** 对参数名称和参数值进行URL编码 **/
public static String generate(String method, Map<String, String> parameter, String accessKeySecret)
throws Exception {
String signString = generateSignString(method, parameter);
byte[] signBytes = HmacSHA1Encrypt(signString, accessKeySecret + "&");
String signature = base64(signBytes);
if ("POST".equals(method)) {
return signature;
}
return URLEncoder.encode(signature, "UTF-8");
}
private static String generateQueryString(TreeMap<String, String> treeMap) {
StringBuilder canonicalizedQueryString = new StringBuilder();
boolean first = true;
for (String key : treeMap.navigableKeySet()) {
String value = treeMap.get(key);
if (!first) {
canonicalizedQueryString.append("&");
}
first = false;
canonicalizedQueryString.append(percentEncode(key)).append("=").append(percentEncode(value));
}
return canonicalizedQueryString.toString();
}
public static String generateSignString(String httpMethod, Map<String, String> parameter) throws IOException {
TreeMap<String, String> sortParameter = new TreeMap<String, String>();
sortParameter.putAll(parameter);
String canonicalizedQueryString = generateQueryString(sortParameter);
if (null == httpMethod) {
throw new RuntimeException("httpMethod can not be empty");
}
/** 构造待签名的字符串* */
StringBuilder stringToSign = new StringBuilder();
stringToSign.append(httpMethod).append("&");
stringToSign.append(percentEncode("/")).append("&");
stringToSign.append(percentEncode(canonicalizedQueryString));
return stringToSign.toString();
}
public static String percentEncode(String value) {
try {
return value == null ? null : URLEncoder.encode(value, ENCODING)
.replaceAll("\\+", "%20")
.replaceAll("\\*", "%2A")
.replaceAll("%7E", "~");
} catch (Exception e) { } catch (Exception e) {
if (retryTime-- <= 0) {
throw new RuntimeException(e);
}
try {
Thread.sleep(100L);
} catch (InterruptedException e1) {
}
}
} }
return "";
} }
} }

25
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
View File

@@ -28,7 +28,7 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
*/ */
public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser, LocalBinLogConnection.FileParserListener { public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser, LocalBinLogConnection.FileParserListener {
private String url = "https://rds.aliyuncs.com/"; // openapi地址 private String url; // openapi地址
private String accesskey; // 云账号的ak private String accesskey; // 云账号的ak
private String secretkey; // 云账号sk private String secretkey; // 云账号sk
private String instanceId; // rds实例id private String instanceId; // rds实例id
@@ -36,7 +36,7 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
private Long endTime; private Long endTime;
private BinlogDownloadQueue binlogDownloadQueue; private BinlogDownloadQueue binlogDownloadQueue;
private ParseFinishListener finishListener; private ParseFinishListener finishListener;
private int batchSize; private int batchFileSize;
public RdsLocalBinlogEventParser(){ public RdsLocalBinlogEventParser(){
} }
@@ -65,10 +65,9 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
instanceId, instanceId,
new Date(startTime), new Date(startTime),
new Date(endTime)); new Date(endTime));
binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchSize, directory); binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchFileSize, directory);
binlogDownloadQueue.silenceDownload(); binlogDownloadQueue.silenceDownload();
needWait = true; needWait = true;
parallel = false;
// try to download one file,use to test server id // try to download one file,use to test server id
binlogDownloadQueue.tryOne(); binlogDownloadQueue.tryOne();
} catch (Throwable e) { } catch (Throwable e) {
@@ -91,10 +90,12 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
binlogDownloadQueue.notifyNotMatch(); binlogDownloadQueue.notifyNotMatch();
try { try {
binlogDownloadQueue.cleanDir(); binlogDownloadQueue.cleanDir();
binlogDownloadQueue.tryOne();
binlogDownloadQueue.prepare(); binlogDownloadQueue.prepare();
} catch (Exception e) { } catch (Throwable e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
try { try {
binlogDownloadQueue.execute(new Runnable() { binlogDownloadQueue.execute(new Runnable() {
@@ -163,8 +164,10 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
} }
// 处理下logManager位点问题 // 处理下logManager位点问题
LogPosition logPosition = logPositionManager.getLatestIndexBy(destination); LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
Long timestamp = 0L;
if (logPosition != null && logPosition.getPostion() != null) {
timestamp = logPosition.getPostion().getTimestamp();
EntryPosition position = logPosition.getPostion(); EntryPosition position = logPosition.getPostion();
if (position != null) {
LogPosition newLogPosition = new LogPosition(); LogPosition newLogPosition = new LogPosition();
String journalName = position.getJournalName(); String journalName = position.getJournalName();
int sepIdx = journalName.indexOf("."); int sepIdx = journalName.indexOf(".");
@@ -181,9 +184,13 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
} }
if (binlogDownloadQueue.isLastFile(fileName)) { if (binlogDownloadQueue.isLastFile(fileName)) {
logger.info("all file parse complete, switch to mysql parser!"); logger.warn("last file : " + fileName + " , timestamp : " + timestamp
+ " , all file parse complete, switch to mysql parser!");
finishListener.onFinish(); finishListener.onFinish();
return; return;
} else {
logger.warn("parse local binlog file : " + fileName + " , timestamp : " + timestamp
+ " , try the next binlog !");
} }
binlogDownloadQueue.prepare(); binlogDownloadQueue.prepare();
} catch (Exception e) { } catch (Exception e) {
@@ -207,7 +214,7 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
void onFinish(); void onFinish();
} }
public void setBatchSize(int batchSize) { public void setBatchFileSize(int batchFileSize) {
this.batchSize = batchSize; this.batchFileSize = batchFileSize;
} }
} }

8
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java
View File

@@ -69,4 +69,12 @@ public class BinlogFile {
public void setFileName(String fileName) { public void setFileName(String fileName) {
this.fileName = fileName; this.fileName = fileName;
} }
@Override
public String toString() {
return "BinlogFile [FileSize=" + FileSize + ", LogBeginTime=" + LogBeginTime + ", LogEndTime=" + LogEndTime
+ ", DownloadLink=" + DownloadLink + ", HostInstanceID=" + HostInstanceID + ", LinkExpiredTime="
+ LinkExpiredTime + ", fileName=" + fileName + "]";
}
} }

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java
View File

@@ -5,6 +5,7 @@ package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
* @since 1.0.25 * @since 1.0.25
*/ */
public class DescribeBinlogFileResult { public class DescribeBinlogFileResult {
private RdsItem Items; private RdsItem Items;
private long PageNumber; private long PageNumber;
private long TotalRecordCount; private long TotalRecordCount;

11
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java
View File

@@ -15,7 +15,8 @@ public class RdsBackupPolicy {
*/ */
private String PreferredBackupTime; private String PreferredBackupTime;
/** /**
* 数据备份周期。Monday：周一；Tuesday：周二；Wednesday：周三；Thursday：周四；Friday：周五；Saturday：周六；Sunday：周日。 * 数据备份周期。Monday：周一；Tuesday：周二；Wednesday：周三；Thursday：周四；Friday：周五；Saturday：
* 周六；Sunday：周日。
*/ */
private String PreferredBackupPeriod; private String PreferredBackupPeriod;
/** /**
@@ -66,4 +67,12 @@ public class RdsBackupPolicy {
public void setLogBackupRetentionPeriod(int logBackupRetentionPeriod) { public void setLogBackupRetentionPeriod(int logBackupRetentionPeriod) {
LogBackupRetentionPeriod = logBackupRetentionPeriod; LogBackupRetentionPeriod = logBackupRetentionPeriod;
} }
@Override
public String toString() {
return "RdsBackupPolicy [BackupRetentionPeriod=" + BackupRetentionPeriod + ", PreferredBackupTime="
+ PreferredBackupTime + ", PreferredBackupPeriod=" + PreferredBackupPeriod + ", BackupLog=" + BackupLog
+ ", LogBackupRetentionPeriod=" + LogBackupRetentionPeriod + "]";
}
} }

7
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java
View File

@@ -7,6 +7,7 @@ import java.util.List;
* @since 1.0.25 * @since 1.0.25
*/ */
public class RdsItem { public class RdsItem {
private List<BinlogFile> BinLogFile; private List<BinlogFile> BinLogFile;
public List<BinlogFile> getBinLogFile() { public List<BinlogFile> getBinLogFile() {
@@ -16,4 +17,10 @@ public class RdsItem {
public void setBinLogFile(List<BinlogFile> binLogFile) { public void setBinLogFile(List<BinlogFile> binLogFile) {
BinLogFile = binLogFile; BinLogFile = binLogFile;
} }
@Override
public String toString() {
return "RdsItem [BinLogFile=" + BinLogFile + "]";
}
} }

15
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
View File

@@ -1,12 +1,18 @@
package com.alibaba.otter.canal.parse.inbound.mysql.rds.request; package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
import io.netty.handler.codec.http.HttpResponseStatus;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.security.cert.CertificateException; import java.security.cert.CertificateException;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.*; import java.util.Date;
import java.util.Map;
import java.util.TimeZone;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.crypto.Mac; import javax.crypto.Mac;
@@ -32,8 +38,6 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContexts; import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import io.netty.handler.codec.http.HttpResponseStatus;
/** /**
* @author chengjin.lyf on 2018/8/7 下午2:26 * @author chengjin.lyf on 2018/8/7 下午2:26
* @since 1.0.25 * @since 1.0.25
@@ -55,7 +59,6 @@ public abstract class AbstractRequest<T> {
/** /**
* api 版本 * api 版本
*
*/ */
private String version; private String version;
@@ -69,7 +72,6 @@ public abstract class AbstractRequest<T> {
private int timeout = (int) TimeUnit.MINUTES.toMillis(1); private int timeout = (int) TimeUnit.MINUTES.toMillis(1);
private Map<String, String> treeMap = new TreeMap(); private Map<String, String> treeMap = new TreeMap();
public void putQueryString(String name, String value) { public void putQueryString(String name, String value) {
@@ -79,12 +81,10 @@ public abstract class AbstractRequest<T> {
treeMap.put(name, value); treeMap.put(name, value);
} }
public void setVersion(String version) { public void setVersion(String version) {
this.version = version; this.version = version;
} }
public void setEndPoint(String endPoint) { public void setEndPoint(String endPoint) {
this.endPoint = endPoint; this.endPoint = endPoint;
} }
@@ -193,6 +193,7 @@ public abstract class AbstractRequest<T> {
* @return * @return
* @throws IOException * @throws IOException
*/ */
@SuppressWarnings("deprecation")
private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception { private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {

5
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
View File

@@ -1,27 +1,26 @@
package com.alibaba.otter.canal.parse.inbound.mysql.rds.request; package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
/** /**
* rds 备份策略查询 * rds 备份策略查询
*
* @author chengjin.lyf on 2018/8/7 下午3:41 * @author chengjin.lyf on 2018/8/7 下午3:41
* @since 1.0.25 * @since 1.0.25
*/ */
public class DescribeBackupPolicyRequest extends AbstractRequest<RdsBackupPolicy> { public class DescribeBackupPolicyRequest extends AbstractRequest<RdsBackupPolicy> {
public DescribeBackupPolicyRequest(){ public DescribeBackupPolicyRequest(){
setVersion("2014-08-15"); setVersion("2014-08-15");
putQueryString("Action", "DescribeBackupPolicy"); putQueryString("Action", "DescribeBackupPolicy");
} }
public void setRdsInstanceId(String rdsInstanceId) { public void setRdsInstanceId(String rdsInstanceId) {
putQueryString("DBInstanceId", rdsInstanceId); putQueryString("DBInstanceId", rdsInstanceId);
} }

7
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java
View File

@@ -2,12 +2,12 @@ package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
import java.util.Date; import java.util.Date;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference; import com.alibaba.fastjson.TypeReference;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
/** /**
* @author chengjin.lyf on 2018/8/7 下午3:41 * @author chengjin.lyf on 2018/8/7 下午3:41
@@ -15,11 +15,9 @@ import com.alibaba.fastjson.TypeReference;
*/ */
public class DescribeBinlogFilesRequest extends AbstractRequest<DescribeBinlogFileResult> { public class DescribeBinlogFilesRequest extends AbstractRequest<DescribeBinlogFileResult> {
public DescribeBinlogFilesRequest(){ public DescribeBinlogFilesRequest(){
setVersion("2014-08-15"); setVersion("2014-08-15");
putQueryString("Action", "DescribeBinlogFiles"); putQueryString("Action", "DescribeBinlogFiles");
} }
public void setRdsInstanceId(String rdsInstanceId) { public void setRdsInstanceId(String rdsInstanceId) {
@@ -49,7 +47,8 @@ public class DescribeBinlogFilesRequest extends AbstractRequest<DescribeBinlogFi
@Override @Override
protected DescribeBinlogFileResult processResult(HttpResponse response) throws Exception { protected DescribeBinlogFileResult processResult(HttpResponse response) throws Exception {
String result = EntityUtils.toString(response.getEntity()); String result = EntityUtils.toString(response.getEntity());
DescribeBinlogFileResult describeBinlogFileResult = JSONObject.parseObject(result, new TypeReference<DescribeBinlogFileResult>() { DescribeBinlogFileResult describeBinlogFileResult = JSONObject.parseObject(result,
new TypeReference<DescribeBinlogFileResult>() {
}); });
return describeBinlogFileResult; return describeBinlogFileResult;
} }

3
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -68,7 +68,8 @@ public class MemoryTableMeta implements TableMetaTSDB {
try { try {
// druid暂时flush privileges语法解析有问题 // druid暂时flush privileges语法解析有问题
if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")) { if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "grant")) {
repository.console(ddl); repository.console(ddl);
} }
} catch (Throwable e) { } catch (Throwable e) {

13
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
View File

@@ -8,6 +8,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker; import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy; import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager; import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
@@ -32,7 +33,6 @@ public class RdsBinlogEventParserProxyTest {
public static final String TBNAME = ""; public static final String TBNAME = "";
public static final String DDL = ""; public static final String DDL = "";
@Test @Test
public void test_timestamp() throws InterruptedException { public void test_timestamp() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000); final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
@@ -41,7 +41,7 @@ public class RdsBinlogEventParserProxyTest {
final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy(); final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
Calendar calendar = Calendar.getInstance(); Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, -1); calendar.add(Calendar.HOUR_OF_DAY, -24 * 4);
final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis()); final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
controller.setSlaveId(3344L); controller.setSlaveId(3344L);
controller.setDetectingEnable(false); controller.setDetectingEnable(false);
@@ -51,8 +51,10 @@ public class RdsBinlogEventParserProxyTest {
controller.setInstanceId(""); controller.setInstanceId("");
controller.setAccesskey(""); controller.setAccesskey("");
controller.setSecretkey(""); controller.setSecretkey("");
controller.setBatchSize(4); controller.setDirectory("/tmp/binlog");
// controller.setRdsOpenApiUrl("https://rds.aliyuncs.com/"); controller.setEventBlackFilter(new AviaterRegexFilter("mysql\\.*"));
controller.setFilterTableError(true);
controller.setBatchFileSize(4);
controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() { controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
@Override @Override
@@ -79,8 +81,8 @@ public class RdsBinlogEventParserProxyTest {
controller.setLogPositionManager(new AbstractLogPositionManager() { controller.setLogPositionManager(new AbstractLogPositionManager() {
private LogPosition logPosition; private LogPosition logPosition;
public void persistLogPosition(String destination, LogPosition logPosition) { public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
this.logPosition = logPosition; this.logPosition = logPosition;
} }
@@ -105,7 +107,6 @@ public class RdsBinlogEventParserProxyTest {
Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp()); Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
} }
// ======================== helper method ======================= // ======================== helper method =======================
private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) { private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {

27
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java
View File

@@ -1,11 +1,14 @@
package com.alibaba.otter.canal.parse.inbound.mysql; package com.alibaba.otter.canal.parse.inbound.mysql;
import java.io.File;
import java.util.Date; import java.util.Date;
import java.util.List;
import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.DateUtils;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi; import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
/** /**
* @author agapple 2017年10月15日 下午2:14:34 * @author agapple 2017年10月15日 下午2:14:34
@@ -13,15 +16,19 @@ import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi;
*/ */
public class RdsBinlogOpenApiTest { public class RdsBinlogOpenApiTest {
@Test
public void testSimple() throws Throwable { public void testSimple() throws Throwable {
Date startTime = DateUtils.parseDate("2017-10-13 20:56:58", new String[] { "yyyy-MM-dd HH:mm:ss" }); Date startTime = DateUtils.parseDate("2018-08-10 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" });
Date endTime = DateUtils.parseDate("2017-10-14 02:57:59", new String[] { "yyyy-MM-dd HH:mm:ss" }); Date endTime = DateUtils.parseDate("2018-08-11 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" });
RdsBinlogOpenApi.downloadBinlogFiles("https://rds.aliyuncs.com/", String url = "https://rds.aliyuncs.com/";
"", String ak = "";
"", String sk = "";
"rm-bp180v4mfjnm157es", String dbInstanceId = "";
startTime,
endTime, RdsBackupPolicy backupPolicy = RdsBinlogOpenApi.queryBinlogBackupPolicy(url, ak, sk, dbInstanceId);
new File("/tmp/binlog/")); System.out.println(backupPolicy);
List<BinlogFile> binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime);
System.out.println(binlogFiles);
} }
} }