Merge pull request #827 from lcybo/metrics_support

Merge master updates

client/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

common/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<artifactId>canal.common</artifactId>

dbsync/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java

@@ -168,9 +168,7 @@ public final class FileLogFetcher extends LogFetcher {
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
      */
     public void close() throws IOException {
-        if (fin != null) {
+        if (fin != null) fin.close();
-            fin.close();
-        }
 
         fin = null;
     }

deployer/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java

@@ -15,7 +15,6 @@ public class CanalConstants {
     public static final String CANAL_ID                          = ROOT + "." + "id";
     public static final String CANAL_IP                          = ROOT + "." + "ip";
     public static final String CANAL_PORT                        = ROOT + "." + "port";
-    public static final String CANAL_METRICS_PULL_PORT           = ROOT + "." + "metrics.pull.port";
     public static final String CANAL_ZKSERVERS                   = ROOT + "." + "zkServers";
     public static final String CANAL_WITHOUT_NETTY               = ROOT + "." + "withoutNetty";
 

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java

@@ -98,14 +98,6 @@ public class CanalController {
         port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT));
         embededCanalServer = CanalServerWithEmbedded.instance();
         embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
-        try {
-            int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT));
-            embededCanalServer.setMetricsPort(metricsPort);
-        } catch (NumberFormatException e) {
-            logger.info("No valid metrics server port found, use default 11112.");
-            embededCanalServer.setMetricsPort(11112);
-        }
-        
         String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
         if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
             canalServer = CanalServerWithNetty.instance();

deployer/src/main/resources/canal.properties

@@ -4,7 +4,6 @@
 canal.id= 1
 canal.ip=
 canal.port=11111
-canal.metrics.pull.port=11112
 canal.zkServers=
 # flush data to zk
 canal.zookeeper.flush.period = 1000
@@ -67,10 +66,6 @@ canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MO
 canal.instance.tsdb.dbUsername=canal
 canal.instance.tsdb.dbPassword=canal
 
-# rds oss binlog account
-canal.instance.rds.accesskey =
-canal.instance.rds.secretkey =
-
 #################################################
 ######### 		destinations		############# 
 #################################################
@@ -87,6 +82,7 @@ canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml
 canal.instance.global.mode = spring 
 canal.instance.global.lazy = false
 #canal.instance.global.manager.address = 127.0.0.1:1099
+#canal.instance.global.spring.xml = classpath:spring/local-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
 canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml

deployer/src/main/resources/example/instance.properties

@@ -12,11 +12,6 @@ canal.instance.master.position=
 canal.instance.master.timestamp=
 canal.instance.master.gtid=
 
-# rds oss binlog
-canal.instance.rds.accesskey=
-canal.instance.rds.secretkey=
-canal.instance.rds.instanceId=
-
 # table meta tsdb info
 canal.instance.tsdb.enable=true
 #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb

deployer/src/main/resources/example/rds_instance.properties

@@ -0,0 +1,32 @@
+#################################################
+# rds openapi binlog
+canal.instance.rds.open.url=https://rds.aliyuncs.com/
+canal.instance.rds.open.accesskey=
+canal.instance.rds.open.secretkey=
+canal.instance.rds.instanceId=
+canal.instance.rds.startTime=
+canal.instance.rds.endTime=
+
+# local binlog dir
+canal.instance.parser.directory=${canal.file.data.dir:../conf}/${canal.instance.destination:}/binlog
+# position info
+canal.instance.master.address=127.0.0.1:3306
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+#canal.instance.tsdb.dbUsername=canal
+#canal.instance.tsdb.dbPassword=canal
+
+# username/password
+canal.instance.dbUsername=canal
+canal.instance.dbPassword=canal
+canal.instance.defaultDatabaseName=test
+canal.instance.connectionCharset=UTF-8
+# table regex
+canal.instance.filter.regex=test\\..*
+# table black regex
+canal.instance.filter.black.regex=
+#################################################

deployer/src/main/resources/logback.xml

@@ -79,7 +79,7 @@
 	</logger>
     
 	<root level="WARN">
-		<!-- <appender-ref ref="STDOUT"/> -->
+		<appender-ref ref="STDOUT"/>
 		<appender-ref ref="CANAL-ROOT" />
 	</root>
 </configuration>

deployer/src/main/resources/spring/base-instance.xml

@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<beans xmlns="http://www.springframework.org/schema/beans"
-	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
-	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
-	xmlns:context="http://www.springframework.org/schema/context"
-	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
-           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
-           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
-           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
-           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
-	default-autowire="byName">
-
-	<!-- properties -->
-	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
-		<property name="ignoreResourceNotFound" value="true" />
-		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
-		<property name="locationNames">
-			<list>
-				<value>classpath:canal.properties</value>
-				<value>classpath:${canal.instance.destination:}/instance.properties</value>
-			</list>
-		</property>
-	</bean>
-	
-	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
-	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
-		<property name="propertyEditorRegistrars">
-			<list>
-				<ref bean="socketAddressEditor" />
-			</list>
-		</property>
-	</bean>
-	
-	<bean id="baseEventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy" abstract="true">
-		<property name="accesskey" value="${canal.instance.rds.accesskey:}" />
-		<property name="secretkey" value="${canal.instance.rds.secretkey:}" />
-		<property name="instanceId" value="${canal.instance.rds.instanceId:}" />
-	</bean>
-</beans>

deployer/src/main/resources/spring/default-instance.xml

@@ -10,7 +10,26 @@
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
 	default-autowire="byName">
 
-	<import resource="classpath:spring/base-instance.xml" />
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
 	
 	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
 		<property name="destination" value="${canal.instance.destination}" />
@@ -65,7 +84,7 @@
 		<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
 	</bean>
 
-	<bean id="eventParser" parent="baseEventParser" >
+	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
 		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->

deployer/src/main/resources/spring/file-instance.xml

@@ -10,7 +10,26 @@
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
 	default-autowire="byName">
 
-	<import resource="classpath:spring/base-instance.xml" />
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
 	
 	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
 		<property name="destination" value="${canal.instance.destination}" />
@@ -51,7 +70,7 @@
 		<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
 	</bean>
 
-	<bean id="eventParser" parent="baseEventParser">
+	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
 		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->

deployer/src/main/resources/spring/group-instance.xml

@@ -10,7 +10,26 @@
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
 	default-autowire="byName">
 	
-	<import resource="classpath:spring/base-instance.xml" />
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
 
 	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
 		<property name="destination" value="${canal.instance.destination}" />
@@ -57,7 +76,7 @@
 		</property>
 	</bean>
 
-	<bean id="eventParser1" parent="baseEventParser">
+	<bean id="eventParser1" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
 		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->
@@ -155,7 +174,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
 	</bean>
 	
-	<bean id="eventParser2" parent="baseEventParser">
+	<bean id="eventParser2" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
 		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->

deployer/src/main/resources/spring/local-instance.xml

@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:context="http://www.springframework.org/schema/context"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
+	default-autowire="byName">
+
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
+		<property name="destination" value="${canal.instance.destination}" />
+		<property name="eventParser">
+			<ref local="eventParser" />
+		</property>
+		<property name="eventSink">
+			<ref local="eventSink" />
+		</property>
+		<property name="eventStore">
+			<ref local="eventStore" />
+		</property>
+		<property name="metaManager">
+			<ref local="metaManager" />
+		</property>
+		<property name="alarmHandler">
+			<ref local="alarmHandler" />
+		</property>
+	</bean>
+	
+	<!-- 报警处理类 -->
+	<bean id="alarmHandler" class="com.alibaba.otter.canal.common.alarm.LogAlarmHandler" />
+	
+	<bean id="metaManager" class="com.alibaba.otter.canal.meta.FileMixedMetaManager">
+		<property name="dataDir" value="${canal.file.data.dir:../conf}" />
+		<property name="period" value="${canal.file.flush.period:1000}" />
+	</bean>
+	
+	<bean id="eventStore" class="com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer">
+		<property name="bufferSize" value="${canal.instance.memory.buffer.size:16384}" />
+		<property name="bufferMemUnit" value="${canal.instance.memory.buffer.memunit:1024}" />
+		<property name="batchMode" value="${canal.instance.memory.batch.mode:MEMSIZE}" />
+		<property name="ddlIsolation" value="${canal.instance.get.ddl.isolation:false}" />
+	</bean>
+	
+	<bean id="eventSink" class="com.alibaba.otter.canal.sink.entry.EntryEventSink">
+		<property name="eventStore" ref="eventStore" />
+		<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
+	</bean>
+
+	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser">
+		<property name="destination" value="${canal.instance.destination}" />			
+		<property name="alarmHandler" ref="alarmHandler" />
+
+		<!-- 解析过滤处理 -->
+		<property name="eventFilter">
+			<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
+				<constructor-arg index="0" value="${canal.instance.filter.regex:.*\..*}" />
+			</bean>
+		</property>
+		
+		<property name="eventBlackFilter">
+			<bean class="com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter" >
+				<constructor-arg index="0" value="${canal.instance.filter.black.regex:}" />
+				<constructor-arg index="1" value="false" />
+			</bean>
+		</property>
+		
+		<!-- 最大事务解析大小，超过该大小后事务将被切分为多个事务投递 -->
+		<property name="transactionSize" value="${canal.instance.transaction.size:1024}" />
+			
+		<!-- 解析编码 -->
+		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
+	
+		<!-- 解析位点记录 -->
+		<property name="logPositionManager">
+			<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
+				<constructor-arg>
+					<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
+				</constructor-arg>
+				<constructor-arg>
+					<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
+						<constructor-arg ref="metaManager"/>
+					</bean>
+				</constructor-arg>
+			</bean>
+		</property>
+		
+		<!-- 解析数据库信息 -->
+		<property name="masterInfo">
+			<bean class="com.alibaba.otter.canal.parse.support.AuthenticationInfo">
+				<property name="address" value="${canal.instance.master.address}" />
+				<property name="username" value="${canal.instance.dbUsername:retl}" />
+				<property name="password" value="${canal.instance.dbPassword:retl}" />
+				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:test}" />
+			</bean>
+		</property>
+		
+		<property name="filterQueryDml" value="${canal.instance.filter.query.dml:false}" />
+		<property name="filterQueryDcl" value="${canal.instance.filter.query.dcl:false}" />
+		<property name="filterQueryDdl" value="${canal.instance.filter.query.ddl:false}" />
+		<property name="useDruidDdlFilter" value="${canal.instance.filter.druid.ddl:true}" />
+		<property name="filterRows" value="${canal.instance.filter.rows:false}" />
+		<property name="filterTableError" value="${canal.instance.filter.table.error:false}" />
+		<property name="needWait" value="${canal.instance.parser.needWait:false}"/>
+		<property name="directory" value="${canal.instance.parser.directory:}"/>
+		
+		<!-- rds相关 -->
+		<property name="url" value="${canal.instance.rds.open.url:}"/>
+		<property name="accesskey" value="${canal.instance.rds.open.accesskey:}"/>
+		<property name="secretkey" value="${canal.instance.rds.open.secretkey:}"/>
+		<property name="instanceId" value="${canal.instance.rds.instanceId:}"/>
+		<property name="startTime" value="${canal.instance.rds.startTime:}"/>
+		<property name="endTime" value="${canal.instance.rds.endTime:}"/>
+		
+		<!--表结构相关-->
+		<property name="enableTsdb" value="${canal.instance.tsdb.enable:true}"/>
+		<property name="tsdbSpringXml" value="${canal.instance.tsdb.spring.xml:}"/>
+		
+		<!-- parallel parser -->
+		<property name="parallel" value="${canal.instance.parser.parallel:true}" />
+		<property name="parallelThreadSize" value="${canal.instance.parser.parallelThreadSize}" />
+		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
+	</bean>
+</beans>

deployer/src/main/resources/spring/memory-instance.xml

@@ -10,7 +10,26 @@
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd"
 	default-autowire="byName">
 	
-	<import resource="classpath:spring/base-instance.xml" />
+	<!-- properties -->
+	<bean class="com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer" lazy-init="false">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE"/><!-- 允许system覆盖 -->
+		<property name="locationNames">
+			<list>
+				<value>classpath:canal.properties</value>
+				<value>classpath:${canal.instance.destination:}/instance.properties</value>
+			</list>
+		</property>
+	</bean>
+	
+	<bean id="socketAddressEditor" class="com.alibaba.otter.canal.instance.spring.support.SocketAddressEditor" />
+	<bean class="org.springframework.beans.factory.config.CustomEditorConfigurer"> 
+		<property name="propertyEditorRegistrars">
+			<list>
+				<ref bean="socketAddressEditor" />
+			</list>
+		</property>
+	</bean>
 
 	<bean id="instance" class="com.alibaba.otter.canal.instance.spring.CanalInstanceWithSpring">
 		<property name="destination" value="${canal.instance.destination}" />
@@ -48,7 +67,7 @@
 		<property name="filterTransactionEntry" value="${canal.instance.filter.transaction.entry:false}"/>
 	</bean>
 
-	<bean id="eventParser" parent="baseEventParser">
+	<bean id="eventParser" class="com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser">
 		<property name="destination" value="${canal.instance.destination}" />
 		<property name="slaveId" value="${canal.instance.mysql.slaveId:0}" />
 		<!-- 心跳配置 -->

docker/Dockerfile

@@ -1,10 +1,36 @@
-FROM canal/osbase:v1
+FROM centos:centos6.7
 
 MAINTAINER agapple (jianghang115@gmail.com)
 
+# install system
+RUN \
+    /bin/cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
+    echo 'root:Hello1234' | chpasswd && \
+    groupadd -r admin && useradd -g admin admin && \
+    yum install -y man && \
+    yum install -y dstat && \
+    yum install -y unzip && \
+    yum install -y nc && \
+    yum install -y openssh-server && \
+    yum install -y tar && \
+    yum install -y which && \
+    yum install -y wget && \
+    yum install -y perl && \
+    yum install -y file && \
+    ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \
+    ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
+    sed -ri 's/session    required     pam_loginuid.so/#session    required     pam_loginuid.so/g' /etc/pam.d/sshd && \
+    sed -i -e 's/^#Port 22$/Port 2222/' /etc/ssh/sshd_config && \
+    mkdir -p /root/.ssh && chown root.root /root && chmod 700 /root/.ssh && \
+    yum install -y cronie && \
+    sed -i '/session required pam_loginuid.so/d' /etc/pam.d/crond && \
+    yum clean all && \
+    true
+
 # install canal
 COPY image/ /tmp/docker/
 COPY canal.deployer-*.tar.gz /home/admin/
+COPY jdk-8-linux-x64.rpm /tmp/
 
 RUN \
     cp -R /tmp/docker/alidata /alidata && \
@@ -13,6 +39,14 @@ RUN \
     cp -R /tmp/docker/admin/* /home/admin/  && \
     /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \
 
+    touch /var/lib/rpm/* && \ 
+    yum -y install /tmp/jdk-8-linux-x64.rpm && \
+    /bin/rm -f /tmp/jdk-8-linux-x64.rpm && \
+
+    echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \
+    echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \
+    /bin/mv /home/admin/bin/clean_log /etc/cron.d && \
+
     mkdir -p /home/admin/canal-server && \
     tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \
     /bin/rm -f /home/admin/canal.deployer-*.tar.gz && \
@@ -30,4 +64,4 @@ EXPOSE 2222 11111 8000 8080
 WORKDIR /home/admin
 
 ENTRYPOINT [ "/alidata/bin/main.sh" ]
-CMD [ "/home/admin/app.sh" ]
+CMD [ "/home/admin/app.sh" ]

docker/base/Dockerfile

@@ -1,41 +0,0 @@
-FROM centos:centos6.10
-
-MAINTAINER agapple (jianghang115@gmail.com)
-
-env DOWNLOAD_LINK="http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.rpm"
-# install system
-RUN \
-    /bin/cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
-    echo 'root:Hello1234' | chpasswd && \
-    groupadd -r admin && useradd -g admin admin && \
-    yum install -y man && \
-    yum install -y dstat && \
-    yum install -y unzip && \
-    yum install -y nc && \
-    yum install -y openssh-server && \
-    yum install -y tar && \
-    yum install -y which && \
-    yum install -y wget && \
-    yum install -y perl && \
-    yum install -y file && \
-    ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \
-    ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
-    sed -ri 's/session    required     pam_loginuid.so/#session    required     pam_loginuid.so/g' /etc/pam.d/sshd && \
-    sed -i -e 's/^#Port 22$/Port 2222/' /etc/ssh/sshd_config && \
-    mkdir -p /root/.ssh && chown root.root /root && chmod 700 /root/.ssh && \
-    yum install -y cronie && \
-    sed -i '/session required pam_loginuid.so/d' /etc/pam.d/crond && \
-    true
-
-RUN \
-    touch /var/lib/rpm/* && \ 
-    wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=xxx; oraclelicense=accept-securebackup-cookie" "$DOWNLOAD_LINK" -O /tmp/jdk-8-linux-x64.rpm && \
-    yum -y install /tmp/jdk-8-linux-x64.rpm && \
-    /bin/rm -f /tmp/jdk-8-linux-x64.rpm && \
-
-    echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile && \
-    echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile && \
-    yum clean all && \
-    true
-
-CMD ["/bin/bash"]

docker/build.sh

@@ -14,16 +14,17 @@ case "`uname`" in
 esac
 BASE=${bin_abs_path}
 
-if [ "$1" == "base" ] ; then
+if [ ! -f $BASE/jdk*.rpm ] ; then
-    docker build --no-cache -t canal/osbase $BASE/base
+    DOWNLOAD_LINK="http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz"
-else 
+    wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=xxx; oraclelicense=accept-securebackup-cookie" "$DOWNLOAD_LINK" -O $BASE/jdk-8-linux-x64.rpm
-    rm -rf $BASE/canal.*.tar.gz ; 
+fi
-    cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ;
+
-    if [ "$1" == "kafka" ] ; then
+cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ;
-	   cp $BASE/../target/canal-kafka-*.tar.gz $BASE/
+
-	   docker build --no-cache -t canal/canal-server $BASE/
+if [ "$1" == "kafka" ] ; then
-    else 
+	cp $BASE/../target/canal-kafka-*.tar.gz $BASE/
-	   cp $BASE/../target/canal.deployer-*.tar.gz $BASE/
+	docker build --no-cache -t canal/canal-server $BASE/
-	   docker build --no-cache -t canal/canal-server $BASE/
+else 
-    fi
+	cp $BASE/../target/canal.deployer-*.tar.gz $BASE/
+	docker build --no-cache -t canal/canal-server $BASE/
 fi

docker/image/admin/app.sh

@@ -86,9 +86,7 @@ function start_canal() {
         echo "multi destination:$destination is not support"
         exit 1;
     else
-        if [ "$destination" != "" ] && [ "$destination" != "example" ] ; then
+        mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination
-            mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination
-        fi 
     fi
     su admin -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1'
     sleep 5

docker/image/admin/health.sh

@@ -1,11 +1,6 @@
 #!/bin/sh
-metrics_port=`perl -le 'print $ENV{"canal.metrics.pull.port"}'`
+CHECK_URL="http://127.0.0.1:8080/metrics"
-if [ "$metrics_port" == "" ]; then
+CHECK_POINT="success"
-	metrics_port="11112"
-fi
-
-CHECK_URL="http://127.0.0.1:$metrics_port/metrics"
-CHECK_POINT="canal"
 CHECK_COUNT=`curl -s --connect-timeout 7 --max-time 7 $CHECK_URL | grep -c $CHECK_POINT`
 if [ $CHECK_COUNT -eq 0 ]; then
     echo "[FAILED]"

docker/run.sh

@@ -10,7 +10,7 @@ function usage() {
     echo "         -e canal.instance.connectionCharset=UTF-8 \\"
     echo "         -e canal.instance.tsdb.enable=true \\"
     echo "         -e canal.instance.gtidon=false \\"
-    echo "         -e canal.instance.filter.regex=.*\\\\\\..* "
+    echo "         -e canal.instance.filter.regex=.*\\..* "
     exit
 }
 

driver/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

example/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java

@@ -265,6 +265,7 @@ public class AbstractCanalClientTest {
                 }
             } catch (UnsupportedEncodingException e) {
             }
+
             builder.append("    type=" + column.getMysqlType());
             if (column.getUpdated()) {
                 builder.append("    update=" + column.getUpdated());

filter/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

instance/core/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 	<artifactId>canal.instance.core</artifactId>

instance/manager/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java

@@ -6,7 +6,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import com.alibaba.otter.canal.meta.FileMixedMetaManager;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,7 +38,12 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
 import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
 import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
 import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-import com.alibaba.otter.canal.parse.index.*;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
+import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
+import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
 import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
 import com.alibaba.otter.canal.sink.entry.EntryEventSink;

instance/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

instance/spring/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

kafka-client/pom.xml

@@ -1,10 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <artifactId>canal</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.0</version>
+        <version>1.0.26-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>com.alibaba.otter</groupId>
@@ -22,7 +24,7 @@
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>
-            <version>1.1.1</version>
+            <version>0.9.0.1</version>
         </dependency>
 
         <!-- junit -->
@@ -106,4 +108,4 @@
             </build>
         </profile>
     </profiles>
-</project>
+</project>

kafka/pom.xml

@@ -1,10 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <artifactId>canal</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.0</version>
+        <version>1.0.26-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>com.alibaba.otter</groupId>
@@ -23,10 +25,11 @@
             <artifactId>snakeyaml</artifactId>
             <version>1.17</version>
         </dependency>
+
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka_2.11</artifactId>
-            <version>1.1.1</version>
+            <version>0.9.0.1</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -34,12 +37,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
-        <dependency>
-            <groupId>org.jboss.netty</groupId>
-            <artifactId>netty</artifactId>
-            <version>3.2.2.Final</version>
-        </dependency>
     </dependencies>
 
     <build>
@@ -140,4 +137,4 @@
             </build>
         </profile>
     </profiles>
-</project>
+</project>

kafka/src/main/java/com/alibaba/otter/canal/kafka/producer/CanalKafkaProducer.java

@@ -21,7 +21,7 @@ import com.alibaba.otter.canal.protocol.Message;
  */
 public class CanalKafkaProducer {
 
-    private static final Logger logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
+    private static final Logger       logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
 
     private Producer<String, Message> producer;
 
@@ -83,8 +83,6 @@ public class CanalKafkaProducer {
             record = new ProducerRecord<String, Message>(topic.getTopic(), message);
         }
         producer.send(record);
-        if (logger.isDebugEnabled()) {
+        logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
-            logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
-        }
     }
 }

kafka/src/main/resources/kafka.yml

@@ -3,7 +3,7 @@ retries: 0
 batchSize: 16384
 lingerMs: 1
 bufferMemory: 33554432
-# canal的批次大小，单位 k，量大建议改为1M
+# canal的批次大小，单位 k
 canalBatchSize: 50
 filterTransactionEntry: true
 
@@ -13,7 +13,7 @@ canalDestinations:
     partition:
     # 一个destination可以对应多个topic
 #    topics:
-#      - topic: example
+#      - topics: example
 #        partition:
 
 

meta/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

parse/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<artifactId>canal.parse</artifactId>

parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java

@@ -1,30 +0,0 @@
-package com.alibaba.otter.canal.parse.exception;
-
-/**
- * @author chengjin.lyf on 2018/7/20 下午2:54
- * @since 1.0.25
- */
-public class PositionNotFoundException extends CanalParseException {
-
-    private static final long serialVersionUID = -7382448928116244017L;
-
-    public PositionNotFoundException(String errorCode) {
-        super(errorCode);
-    }
-
-    public PositionNotFoundException(String errorCode, Throwable cause) {
-        super(errorCode, cause);
-    }
-
-    public PositionNotFoundException(String errorCode, String errorDesc) {
-        super(errorCode, errorDesc);
-    }
-
-    public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause) {
-        super(errorCode, errorDesc, cause);
-    }
-
-    public PositionNotFoundException(Throwable cause) {
-        super(cause);
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java

@@ -1,32 +0,0 @@
-package com.alibaba.otter.canal.parse.exception;
-
-import com.alibaba.otter.canal.common.CanalException;
-
-/**
- * @author chengjin.lyf on 2018/8/8 下午1:07
- * @since 1.0.25
- */
-public class ServerIdNotMatchException extends CanalException{
-
-    private static final long serialVersionUID = -6124989280379293953L;
-
-    public ServerIdNotMatchException(String errorCode) {
-        super(errorCode);
-    }
-
-    public ServerIdNotMatchException(String errorCode, Throwable cause) {
-        super(errorCode, cause);
-    }
-
-    public ServerIdNotMatchException(String errorCode, String errorDesc) {
-        super(errorCode, errorDesc);
-    }
-
-    public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause) {
-        super(errorCode, errorDesc, cause);
-    }
-
-    public ServerIdNotMatchException(Throwable cause) {
-        super(cause);
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java

@@ -21,7 +21,6 @@ import com.alibaba.otter.canal.filter.CanalEventFilter;
 import com.alibaba.otter.canal.parse.CanalEventParser;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
-import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
 import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
 import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
 import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
@@ -95,8 +94,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
                                                                                     .availableProcessors() * 60 / 100;     // 60%的能力跑解析,剩余部分处理网络
     protected int                                    parallelBufferSize         = 256;                                     // 必须为2的幂
     protected MultiStageCoprocessor                  multiStageCoprocessor;
-    protected ParserExceptionHandler                 parserExceptionHandler;
+
-    protected long                                   serverId;
 
     protected abstract BinlogParser buildParser();
 
@@ -173,16 +171,11 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
                         preDump(erosaConnection);
 
                         erosaConnection.connect();// 链接
-
-                        long queryServerId = erosaConnection.queryServerId();
-                        if (queryServerId != 0) {
-                            serverId = queryServerId;
-                        }
                         // 4. 获取最后的位置信息
                         EntryPosition position = findStartPosition(erosaConnection);
                         final EntryPosition startPosition = position;
                         if (startPosition == null) {
-                            throw new PositionNotFoundException("can't find start position for " + destination);
+                            throw new CanalParseException("can't find start position for " + destination);
                         }
 
                         if (!processTableMeta(startPosition)) {
@@ -285,9 +278,6 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
                                 runningInfo.getAddress().toString()), e);
                             sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
                         }
-                        if (parserExceptionHandler != null) {
-                            parserExceptionHandler.handle(e);
-                        }
                     } finally {
                         // 重新置为中断状态
                         Thread.interrupted();
@@ -437,7 +427,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
         return logPosition;
     }
 
-    protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, Long startPosition) {
+    protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, long startPosition) {
         if (lastPosition != null) {
             logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
                 lastPosition.getPostion()),
@@ -626,20 +616,5 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
         this.parallelBufferSize = parallelBufferSize;
     }
 
-    public ParserExceptionHandler getParserExceptionHandler() {
-        return parserExceptionHandler;
-    }
-
-    public void setParserExceptionHandler(ParserExceptionHandler parserExceptionHandler) {
-        this.parserExceptionHandler = parserExceptionHandler;
-    }
-
-    public long getServerId() {
-        return serverId;
-    }
-
-    public void setServerId(long serverId) {
-        this.serverId = serverId;
-    }
 
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java

@@ -40,6 +40,4 @@ public interface ErosaConnection {
     public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException;
 
     ErosaConnection fork();
-
-    public long queryServerId() throws IOException;
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java

@@ -2,7 +2,6 @@ package com.alibaba.otter.canal.parse.inbound;
 
 import com.alibaba.otter.canal.common.CanalLifeCycle;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
-import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * 针对解析器提供一个多阶段协同的处理
@@ -24,7 +23,7 @@ public interface MultiStageCoprocessor extends CanalLifeCycle {
      */
     public boolean publish(LogBuffer buffer);
 
-    public boolean publish(LogEvent event);
+    public boolean publish(LogBuffer buffer, String binlogFileName);
 
     public void reset();
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java

@@ -1,9 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound;
-
-/**
- * @author chengjin.lyf on 2018/7/20 下午3:55
- * @since 1.0.25
- */
-public interface ParserExceptionHandler {
-    void handle(Throwable e);
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java

@@ -134,10 +134,10 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
 
     protected MultiStageCoprocessor buildMultiStageCoprocessor() {
         MysqlMultiStageCoprocessor mysqlMultiStageCoprocessor = new MysqlMultiStageCoprocessor(parallelBufferSize,
-            parallelThreadSize,
+                parallelThreadSize,
-            (LogEventConvert) binlogParser,
+                (LogEventConvert) binlogParser,
-            transactionBuffer,
+                transactionBuffer,
-            destination);
+                destination);
         mysqlMultiStageCoprocessor.setEventsPublishBlockingTime(eventsPublishBlockingTime);
         return mysqlMultiStageCoprocessor;
     }
@@ -204,12 +204,12 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         }
     }
 
+    public AtomicLong getEventsPublishBlockingTime() {
+        return this.eventsPublishBlockingTime;
+    }
+    
     public void setTableMetaTSDBFactory(TableMetaTSDBFactory tableMetaTSDBFactory) {
         this.tableMetaTSDBFactory = tableMetaTSDBFactory;
     }
 
-    public AtomicLong getEventsPublishBlockingTime() {
-        return this.eventsPublishBlockingTime;
-    }
-
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java

@@ -10,18 +10,18 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
-import com.alibaba.otter.canal.parse.exception.CanalParseException;
-import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
 import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
 import com.alibaba.otter.canal.parse.inbound.MultiStageCoprocessor;
 import com.alibaba.otter.canal.parse.inbound.SinkFunction;
 import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue;
 import com.taobao.tddl.dbsync.binlog.FileLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 import com.taobao.tddl.dbsync.binlog.LogPosition;
 import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
 
 /**
  * local bin log connection (not real connection)
@@ -36,8 +36,6 @@ public class LocalBinLogConnection implements ErosaConnection {
     private String              directory;
     private int                 bufferSize = 16 * 1024;
     private boolean             running    = false;
-    private long                serverId;
-    private FileParserListener  parserListener;
 
     public LocalBinLogConnection(){
     }
@@ -90,14 +88,14 @@ public class LocalBinLogConnection implements ErosaConnection {
             while (running) {
                 boolean needContinue = true;
                 LogEvent event = null;
+                // 处理一下binlog文件名
+                event = new RotateLogEvent(context.getLogPosition().getFileName(), 4);
+                func.sink(event);
                 while (fetcher.fetch()) {
                     event = decoder.decode(fetcher, context);
                     if (event == null) {
                         continue;
                     }
-                    if (serverId != 0 && event.getServerId() != serverId) {
-                        throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
-                    }
 
                     if (!func.sink(event)) {
                         needContinue = false;
@@ -105,9 +103,8 @@ public class LocalBinLogConnection implements ErosaConnection {
                     }
                 }
 
-                fetcher.close(); // 关闭上一个文件
-                parserFinish(current.getName());
                 if (needContinue) {// 读取下一个
+                    fetcher.close(); // 关闭上一个文件
 
                     File nextFile;
                     if (needWait) {
@@ -163,10 +160,6 @@ public class LocalBinLogConnection implements ErosaConnection {
                 while (fetcher.fetch()) {
                     LogEvent event = decoder.decode(fetcher, context);
                     if (event != null) {
-                        if (serverId != 0 && event.getServerId() != serverId) {
-                            throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
-                        }
-
                         if (event.getWhen() > timestampSeconds) {
                             break;
                         }
@@ -183,9 +176,6 @@ public class LocalBinLogConnection implements ErosaConnection {
                         } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
                             lastXidLogFilename = current.getName();
                             lastXidLogFileOffset = event.getLogPos();
-                        } else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
-                            lastXidLogFilename = current.getName();
-                            lastXidLogFileOffset = event.getLogPos();
                         }
                     }
                 }
@@ -222,37 +212,25 @@ public class LocalBinLogConnection implements ErosaConnection {
     @Override
     public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
         File current = new File(directory, binlogfilename);
-        if (!current.exists()) {
-            throw new CanalParseException("binlog:" + binlogfilename + " is not found");
-        }
 
         FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
-        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
-        LogContext context = new LogContext();
         try {
             fetcher.open(current, binlogPosition);
-            context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
             while (running) {
                 boolean needContinue = true;
-                LogEvent event = null;
                 while (fetcher.fetch()) {
-                    event = decoder.decode(fetcher, context);
+                    LogBuffer buffer = fetcher.duplicate();
-                    if (event == null) {
+                    fetcher.consume(fetcher.limit());
-                        continue;
+                    // set filename
-                    }
+                    if (!coprocessor.publish(buffer, binlogfilename)) {
-                    if (serverId != 0 && event.getServerId() != serverId) {
-                        throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
-                    }
-
-                    if (!coprocessor.publish(event)) {
                         needContinue = false;
                         break;
                     }
                 }
 
-                fetcher.close(); // 关闭上一个文件
-                parserFinish(binlogfilename);
                 if (needContinue) {// 读取下一个
+                    fetcher.close(); // 关闭上一个文件
+
                     File nextFile;
                     if (needWait) {
                         nextFile = binlogs.waitForNextFile(current);
@@ -280,12 +258,6 @@ public class LocalBinLogConnection implements ErosaConnection {
         }
     }
 
-    private void parserFinish(String fileName) {
-        if (parserListener != null) {
-            parserListener.onFinish(fileName);
-        }
-    }
-
     @Override
     public void dump(long timestampMills, MultiStageCoprocessor coprocessor) throws IOException {
         List<File> currentBinlogs = binlogs.currentBinlogs();
@@ -314,10 +286,6 @@ public class LocalBinLogConnection implements ErosaConnection {
                 while (fetcher.fetch()) {
                     LogEvent event = decoder.decode(fetcher, context);
                     if (event != null) {
-                        if (serverId != 0 && event.getServerId() != serverId) {
-                            throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
-                        }
-
                         if (event.getWhen() > timestampSeconds) {
                             break;
                         }
@@ -334,9 +302,6 @@ public class LocalBinLogConnection implements ErosaConnection {
                         } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
                             lastXidLogFilename = current.getName();
                             lastXidLogFileOffset = event.getLogPos();
-                        } else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
-                            lastXidLogFilename = current.getName();
-                            lastXidLogFileOffset = event.getLogPos();
                         }
                     }
                 }
@@ -379,11 +344,6 @@ public class LocalBinLogConnection implements ErosaConnection {
         return connection;
     }
 
-    @Override
-    public long queryServerId() {
-        return 0;
-    }
-
     public boolean isNeedWait() {
         return needWait;
     }
@@ -408,21 +368,4 @@ public class LocalBinLogConnection implements ErosaConnection {
         this.bufferSize = bufferSize;
     }
 
-    public long getServerId() {
-        return serverId;
-    }
-
-    public void setServerId(long serverId) {
-        this.serverId = serverId;
-    }
-
-    public void setParserListener(FileParserListener parserListener) {
-        this.parserListener = parserListener;
-    }
-
-    public interface FileParserListener {
-
-        void onFinish(String fileName);
-    }
-
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java

@@ -10,7 +10,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,20 +38,20 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 public class MysqlConnection implements ErosaConnection {
 
-    private static final Logger logger      = LoggerFactory.getLogger(MysqlConnection.class);
+    private static final Logger       logger                    = LoggerFactory.getLogger(MysqlConnection.class);
 
-    private MysqlConnector      connector;
+    private MysqlConnector            connector;
-    private long                slaveId;
+    private long                      slaveId;
-    private Charset             charset     = Charset.forName("UTF-8");
+    private Charset                   charset                   = Charset.forName("UTF-8");
-    private BinlogFormat        binlogFormat;
+    private BinlogFormat              binlogFormat;
-    private BinlogImage         binlogImage;
+    private BinlogImage               binlogImage;
 
     // tsdb releated
-    private AuthenticationInfo  authInfo;
+    private AuthenticationInfo        authInfo;
-    protected int               connTimeout = 5 * 1000;                                      // 5秒
+    protected     int                 connTimeout               = 5 * 1000;                                      // 5秒
-    protected int               soTimeout   = 60 * 60 * 1000;                                // 1小时
+    protected     int                 soTimeout                 = 60 * 60 * 1000;                                // 1小时
     // dump binlog bytes, 暂不包括meta与TSDB
-    private AtomicLong          receivedBinlogBytes;
+    private AtomicLong                receivedBinlogBytes;
 
     public MysqlConnection(){
     }
@@ -333,16 +332,6 @@ public class MysqlConnection implements ErosaConnection {
         return connection;
     }
 
-    @Override
-    public long queryServerId() throws IOException {
-        ResultSetPacket resultSetPacket = query("show variables like 'server_id'");
-        List<String> fieldValues = resultSetPacket.getFieldValues();
-        if (fieldValues == null || fieldValues.size() != 2) {
-            return 0;
-        }
-        return NumberUtils.toLong(fieldValues.get(1));
-    }
-
     // ====================== help method ====================
 
     /**
@@ -477,6 +466,8 @@ public class MysqlConnection implements ErosaConnection {
         }
     }
 
+
+
     public static enum BinlogFormat {
 
         STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED");

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java

@@ -47,32 +47,32 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  */
 public class MysqlEventParser extends AbstractMysqlEventParser implements CanalEventParser, CanalHASwitchable {
 
-    private CanalHAController    haController                      = null;
+    private CanalHAController  haController                      = null;
 
-    private int                  defaultConnectionTimeoutInSeconds = 30;                // sotimeout
+    private int                defaultConnectionTimeoutInSeconds = 30;       // sotimeout
-    private int                  receiveBufferSize                 = 64 * 1024;
+    private int                receiveBufferSize                 = 64 * 1024;
-    private int                  sendBufferSize                    = 64 * 1024;
+    private int                sendBufferSize                    = 64 * 1024;
     // 数据库信息
-    protected AuthenticationInfo masterInfo;                                            // 主库
+    private AuthenticationInfo masterInfo;                                   // 主库
-    protected AuthenticationInfo standbyInfo;                                           // 备库
+    private AuthenticationInfo standbyInfo;                                  // 备库
     // binlog信息
-    protected EntryPosition      masterPosition;
+    private EntryPosition      masterPosition;
-    protected EntryPosition      standbyPosition;
+    private EntryPosition      standbyPosition;
-    private long                 slaveId;                                               // 链接到mysql的slave
+    private long               slaveId;                                      // 链接到mysql的slave
     // 心跳检查信息
-    private String               detectingSQL;                                          // 心跳sql
+    private String             detectingSQL;                                 // 心跳sql
-    private MysqlConnection      metaConnection;                                        // 查询meta信息的链接
+    private MysqlConnection    metaConnection;                               // 查询meta信息的链接
-    private TableMetaCache       tableMetaCache;                                        // 对应meta
+    private TableMetaCache     tableMetaCache;                               // 对应meta
-    private int                  fallbackIntervalInSeconds         = 60;                // 切换回退时间
+    private int                fallbackIntervalInSeconds         = 60;       // 切换回退时间
-    private BinlogFormat[]       supportBinlogFormats;                                  // 支持的binlogFormat,如果设置会执行强校验
+    private BinlogFormat[]     supportBinlogFormats;                         // 支持的binlogFormat,如果设置会执行强校验
-    private BinlogImage[]        supportBinlogImages;                                   // 支持的binlogImage,如果设置会执行强校验
+    private BinlogImage[]      supportBinlogImages;                          // 支持的binlogImage,如果设置会执行强校验
 
     // update by yishun.chen,特殊异常处理参数
-    private int                  dumpErrorCount                    = 0;                 // binlogDump失败异常计数
+    private       int                 dumpErrorCount            = 0;        // binlogDump失败异常计数
-    private int                  dumpErrorCountThreshold           = 2;                 // binlogDump失败异常计数阀值
+    private       int                 dumpErrorCountThreshold   = 2;        // binlogDump失败异常计数阀值
 
     // instance received binlog bytes
-    private final AtomicLong     receivedBinlogBytes               = new AtomicLong(0L);
+    private final AtomicLong          receivedBinlogBytes       = new AtomicLong(0L);
 
     protected ErosaConnection buildErosaConnection() {
         return buildMysqlConnection(this.runningInfo);
@@ -756,9 +756,16 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
                         Long logposTimestamp = entry.getHeader().getExecuteTime();
                         Long serverId = entry.getHeader().getServerId();
 
-                        // 如果最小的一条记录都不满足条件，可直接退出
+                        if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())
-                        if (logposTimestamp >= startTimestamp) {
+                            || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
-                            return false;
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] {
+                                        logfilename, logfileoffset, logposTimestamp, startTimestamp });
+                            }
+                            // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
+                            if (logposTimestamp >= startTimestamp) {
+                                return false;
+                            }
                         }
 
                         if (StringUtils.equals(endPosition.getJournalName(), logfilename)
@@ -907,6 +914,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         this.dumpErrorCountThreshold = dumpErrorCountThreshold;
     }
 
+
+
     public AtomicLong getReceivedBinlogBytes() {
         return this.receivedBinlogBytes;
     }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java

@@ -5,6 +5,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.LockSupport;
 
+import org.apache.commons.lang.StringUtils;
+
 import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
 import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
@@ -30,6 +32,7 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogDecoder;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.LogPosition;
 import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
@@ -50,7 +53,7 @@ import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
  */
 public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implements MultiStageCoprocessor {
 
-    private static final int             maxFullTimes = 10;
+    private static final int             maxFullTimes               = 10;
     private LogEventConvert              logEventConvert;
     private EventTransactionBuffer       transactionBuffer;
     private ErosaConnection              connection;
@@ -138,18 +141,14 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         super.stop();
     }
 
-    public boolean publish(LogBuffer buffer) {
-        return this.publish(buffer, null);
-    }
-
     /**
      * 网络数据投递
      */
-    public boolean publish(LogEvent event) {
+    public boolean publish(LogBuffer buffer) {
-        return this.publish(null, event);
+        return publish(buffer, null);
     }
 
-    private boolean publish(LogBuffer buffer, LogEvent event) {
+    public boolean publish(LogBuffer buffer, String binlogFileName) {
         if (!isStart()) {
             if (exception != null) {
                 throw exception;
@@ -170,11 +169,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         do {
             try {
                 long next = disruptorMsgBuffer.tryNext();
-                MessageEvent data = disruptorMsgBuffer.get(next);
+                MessageEvent event = disruptorMsgBuffer.get(next);
-                if (buffer != null) {
+                event.setBuffer(buffer);
-                    data.setBuffer(buffer);
+                if (binlogFileName != null) {
-                } else {
+                    event.setBinlogFileName(binlogFileName);
-                    data.setEvent(event);
                 }
                 disruptorMsgBuffer.publish(next);
                 if (fullTimes > 0) {
@@ -186,7 +184,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
                     blockingStart = System.nanoTime();
                 }
                 // park
-                // LockSupport.parkNanos(1L);
+                //LockSupport.parkNanos(1L);
                 applyWait(++fullTimes);
                 interupted = Thread.interrupted();
                 if (fullTimes % 1000 == 0) {
@@ -219,6 +217,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         start();
     }
 
+
     private class SimpleParserStage implements EventHandler<MessageEvent>, LifecycleAware {
 
         private LogDecoder decoder;
@@ -231,13 +230,17 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
         public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
             try {
-                LogEvent logEvent = event.getEvent();
+                LogBuffer buffer = event.getBuffer();
-                if (logEvent == null) {
+                if (StringUtils.isNotEmpty(event.getBinlogFileName())
-                    LogBuffer buffer = event.getBuffer();
+                    && !context.getLogPosition().getFileName().equals(event.getBinlogFileName())) {
-                    logEvent = decoder.decode(buffer, context);
+                    // set roate binlog file name
-                    event.setEvent(logEvent);
+                    context.setLogPosition(new LogPosition(event.getBinlogFileName(), context.getLogPosition()
+                        .getPosition()));
                 }
 
+                LogEvent logEvent = decoder.decode(buffer, context);
+                event.setEvent(logEvent);
+
                 int eventType = logEvent.getHeader().getType();
                 TableMeta tableMeta = null;
                 boolean needDmlParse = false;
@@ -338,6 +341,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
                 // clear for gc
                 event.setBuffer(null);
+                event.setBinlogFileName(null);
                 event.setEvent(null);
                 event.setTable(null);
                 event.setEntry(null);
@@ -361,12 +365,21 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
     class MessageEvent {
 
+        private String           binlogFileName;      // for local binlog parse
         private LogBuffer        buffer;
         private CanalEntry.Entry entry;
         private boolean          needDmlParse = false;
         private TableMeta        table;
         private LogEvent         event;
 
+        public String getBinlogFileName() {
+            return binlogFileName;
+        }
+
+        public void setBinlogFileName(String binlogFileName) {
+            this.binlogFileName = binlogFileName;
+        }
+
         public LogBuffer getBuffer() {
             return buffer;
         }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java

@@ -10,6 +10,7 @@ import java.util.Arrays;
 import java.util.BitSet;
 import java.util.List;
 
+import com.taobao.tddl.dbsync.binlog.event.*;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -40,23 +41,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry.Type;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
 import com.google.protobuf.ByteString;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
-import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.LogHeader;
-import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
-import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
-import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 
 /**
@@ -887,14 +872,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
         try {
             return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             String message = ExceptionUtils.getRootCauseMessage(e);
             if (filterTableError) {
                 if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                     return null;
-                } else if (StringUtils.contains(message, "errorNumber=1142")
-                           && StringUtils.contains(message, "command denied")) {
-                    return null;
                 }
             }
 

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java

@@ -25,14 +25,13 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException;
  */
 public class BinLogFileQueue {
 
-    private String              baseName       = "mysql-bin.";
+    private String        baseName       = "mysql-bin.";
-    private List<File>          binlogs        = new ArrayList<File>();
+    private List<File>    binlogs        = new ArrayList<File>();
-    private File                directory;
+    private File          directory;
-    private ReentrantLock       lock           = new ReentrantLock();
+    private ReentrantLock lock           = new ReentrantLock();
-    private Condition           nextCondition  = lock.newCondition();
+    private Condition     nextCondition  = lock.newCondition();
-    private Timer               timer          = new Timer(true);
+    private Timer         timer          = new Timer(true);
-    private long                reloadInterval = 10 * 1000L;           // 10秒
+    private long          reloadInterval = 10 * 1000L;           // 10秒
-    private CanalParseException exception      = null;
 
     public BinLogFileQueue(String directory){
         this(new File(directory));
@@ -53,24 +52,9 @@ public class BinLogFileQueue {
         timer.scheduleAtFixedRate(new TimerTask() {
 
             public void run() {
-                try {
+                List<File> files = listBinlogFiles();
-                    // File errorFile = new File(BinLogFileQueue.this.directory,
+                for (File file : files) {
-                    // errorFileName);
+                    offer(file);
-                    // if (errorFile.isFile() && errorFile.exists()) {
-                    // String text = StringUtils.join(IOUtils.readLines(new
-                    // FileInputStream(errorFile)), "\n");
-                    // exception = new CanalParseException(text);
-                    // }
-                    List<File> files = listBinlogFiles();
-                    for (File file : files) {
-                        offer(file);
-                    }
-                } catch (Throwable e) {
-                    exception = new CanalParseException(e);
-                }
-
-                if (exception != null) {
-                    offer(null);
                 }
             }
         }, reloadInterval, reloadInterval);
@@ -85,10 +69,6 @@ public class BinLogFileQueue {
     public File getNextFile(File pre) {
         try {
             lock.lockInterruptibly();
-            if (exception != null) {
-                throw exception;
-            }
-
             if (binlogs.size() == 0) {
                 return null;
             } else {
@@ -114,10 +94,6 @@ public class BinLogFileQueue {
     public File getBefore(File file) {
         try {
             lock.lockInterruptibly();
-            if (exception != null) {
-                throw exception;
-            }
-
             if (binlogs.size() == 0) {
                 return null;
             } else {
@@ -154,9 +130,6 @@ public class BinLogFileQueue {
                 nextCondition.await();// 等待新文件
             }
 
-            if (exception != null) {
-                throw exception;
-            }
             if (pre == null) {// 第一次
                 return binlogs.get(0);
             } else {
@@ -197,16 +170,13 @@ public class BinLogFileQueue {
     private boolean offer(File file) {
         try {
             lock.lockInterruptibly();
-            if (file != null) {
+            if (!binlogs.contains(file)) {
-                if (!binlogs.contains(file)) {
+                binlogs.add(file);
-                    binlogs.add(file);
+                nextCondition.signalAll();// 唤醒
-                    nextCondition.signalAll();// 唤醒
+                return true;
-                    return true;
+            } else {
-                }
+                return false;
             }
-
-            nextCondition.signalAll();// 唤醒
-            return false;
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
             return false;

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java

@@ -1,339 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds;
-
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.LockSupport;
-
-import javax.net.ssl.SSLContext;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.ssl.SSLContextBuilder;
-import org.apache.http.ssl.TrustStrategy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午3:10
- * @since 1.0.25
- */
-public class BinlogDownloadQueue {
-
-    private static final Logger             logger        = LoggerFactory.getLogger(BinlogDownloadQueue.class);
-    private static final int                TIMEOUT       = 10000;
-
-    private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<BinlogFile>();
-    private LinkedBlockingQueue<Runnable>   taskQueue     = new LinkedBlockingQueue<Runnable>();
-    private LinkedList<BinlogFile>          binlogList;
-    private final int                       batchFileSize;
-    private Thread                          downloadThread;
-    public boolean                          running       = true;
-    private final String                    destDir;
-    private String                          hostId;
-    private int                             currentSize;
-    private String                          lastDownload;
-
-    public BinlogDownloadQueue(List<BinlogFile> downloadQueue, int batchFileSize, String destDir) throws IOException{
-        this.binlogList = new LinkedList(downloadQueue);
-        this.batchFileSize = batchFileSize;
-        this.destDir = destDir;
-        this.currentSize = 0;
-        prepareBinlogList();
-        cleanDir();
-    }
-
-    private void prepareBinlogList() {
-        for (BinlogFile binlog : this.binlogList) {
-            String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?");
-            binlog.setFileName(fileName);
-        }
-        Collections.sort(this.binlogList, new Comparator<BinlogFile>() {
-
-            @Override
-            public int compare(BinlogFile o1, BinlogFile o2) {
-                return o1.getFileName().compareTo(o2.getFileName());
-            }
-        });
-    }
-
-    public void cleanDir() throws IOException {
-        File destDirFile = new File(destDir);
-        FileUtils.forceMkdir(destDirFile);
-        FileUtils.cleanDirectory(destDirFile);
-    }
-
-    public void silenceDownload() {
-        if (downloadThread != null) {
-            return;
-        }
-        downloadThread = new Thread(new DownloadThread(), "download-" + destDir);
-        downloadThread.setDaemon(true);
-        downloadThread.start();
-    }
-
-    public BinlogFile tryOne() throws Throwable {
-        BinlogFile binlogFile = binlogList.poll();
-        download(binlogFile);
-        hostId = binlogFile.getHostInstanceID();
-        this.currentSize++;
-        return binlogFile;
-    }
-
-    public void notifyNotMatch() {
-        this.currentSize--;
-        filter(hostId);
-    }
-
-    private void filter(String hostInstanceId) {
-        Iterator<BinlogFile> it = binlogList.iterator();
-        while (it.hasNext()) {
-            BinlogFile bf = it.next();
-            if (bf.getHostInstanceID().equalsIgnoreCase(hostInstanceId)) {
-                it.remove();
-            } else {
-                hostId = bf.getHostInstanceID();
-            }
-        }
-    }
-
-    public boolean isLastFile(String fileName) {
-        String needCompareName = lastDownload;
-        if (StringUtils.isNotEmpty(needCompareName) && StringUtils.endsWith(needCompareName, "tar")) {
-            needCompareName = needCompareName.substring(0, needCompareName.indexOf("."));
-        }
-        return fileName.equalsIgnoreCase(needCompareName) && binlogList.isEmpty();
-    }
-
-    public void prepare() throws InterruptedException {
-        for (int i = this.currentSize; i < batchFileSize && !binlogList.isEmpty(); i++) {
-            BinlogFile binlogFile = null;
-            while (!binlogList.isEmpty()) {
-                binlogFile = binlogList.poll();
-                if (!binlogFile.getHostInstanceID().equalsIgnoreCase(hostId)) {
-                    continue;
-                }
-                break;
-            }
-            if (binlogFile == null) {
-                break;
-            }
-            this.downloadQueue.put(binlogFile);
-            this.lastDownload = "mysql-bin." + binlogFile.getFileName();
-            this.currentSize++;
-        }
-    }
-
-    public void downOne() {
-        this.currentSize--;
-    }
-
-    public void release() {
-        running = false;
-        this.currentSize = 0;
-        binlogList.clear();
-        downloadQueue.clear();
-    }
-
-    private void download(BinlogFile binlogFile) throws Throwable {
-        String downloadLink = binlogFile.getDownloadLink();
-        String fileName = binlogFile.getFileName();
-
-        downloadLink = downloadLink.trim();
-        CloseableHttpClient httpClient = null;
-        if (downloadLink.startsWith("https")) {
-            HttpClientBuilder builder = HttpClientBuilder.create();
-            builder.setMaxConnPerRoute(50);
-            builder.setMaxConnTotal(100);
-            // 创建支持忽略证书的https
-            final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
-
-                @Override
-                public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
-                    return true;
-                }
-            }).build();
-
-            httpClient = HttpClientBuilder.create()
-                .setSSLContext(sslContext)
-                .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
-                    .register("http", PlainConnectionSocketFactory.INSTANCE)
-                    .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
-                    .build()))
-                .build();
-        } else {
-            httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
-        }
-
-        HttpGet httpGet = new HttpGet(downloadLink);
-        RequestConfig requestConfig = RequestConfig.custom()
-            .setConnectTimeout(TIMEOUT)
-            .setConnectionRequestTimeout(TIMEOUT)
-            .setSocketTimeout(TIMEOUT)
-            .build();
-        httpGet.setConfig(requestConfig);
-        HttpResponse response = httpClient.execute(httpGet);
-        int statusCode = response.getStatusLine().getStatusCode();
-        if (statusCode != HttpResponseStatus.OK.code()) {
-            throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
-        }
-        saveFile(new File(destDir), "mysql-bin." + fileName, response);
-    }
-
-    private static void saveFile(File parentFile, String fileName, HttpResponse response) throws IOException {
-        InputStream is = response.getEntity().getContent();
-        long totalSize = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
-        if (response.getFirstHeader("Content-Disposition") != null) {
-            fileName = response.getFirstHeader("Content-Disposition").getValue();
-            fileName = StringUtils.substringAfter(fileName, "filename=");
-        }
-        boolean isTar = StringUtils.endsWith(fileName, ".tar");
-        FileUtils.forceMkdir(parentFile);
-        FileOutputStream fos = null;
-        try {
-            if (isTar) {
-                TarArchiveInputStream tais = new TarArchiveInputStream(is);
-                TarArchiveEntry tarArchiveEntry = null;
-                while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
-                    String name = tarArchiveEntry.getName();
-                    File tarFile = new File(parentFile, name + ".tmp");
-                    logger.info("start to download file " + tarFile.getName());
-                    if (tarFile.exists()) {
-                        tarFile.delete();
-                    }
-                    BufferedOutputStream bos = null;
-                    try {
-                        bos = new BufferedOutputStream(new FileOutputStream(tarFile));
-                        int read = -1;
-                        byte[] buffer = new byte[1024];
-                        while ((read = tais.read(buffer)) != -1) {
-                            bos.write(buffer, 0, read);
-                        }
-                        logger.info("download file " + tarFile.getName() + " end!");
-                        tarFile.renameTo(new File(parentFile, name));
-                    } finally {
-                        IOUtils.closeQuietly(bos);
-                    }
-                }
-                tais.close();
-            } else {
-                File file = new File(parentFile, fileName + ".tmp");
-                if (file.exists()) {
-                    file.delete();
-                }
-
-                if (!file.isFile()) {
-                    file.createNewFile();
-                }
-                try {
-                    fos = new FileOutputStream(file);
-                    byte[] buffer = new byte[1024];
-                    int len;
-                    long copySize = 0;
-                    long nextPrintProgress = 0;
-                    logger.info("start to download file " + file.getName());
-                    while ((len = is.read(buffer)) != -1) {
-                        fos.write(buffer, 0, len);
-                        copySize += len;
-                        long progress = copySize * 100 / totalSize;
-                        if (progress >= nextPrintProgress) {
-                            logger.info("download " + file.getName() + " progress : " + progress
-                                        + "% , download size : " + copySize + ", total size : " + totalSize);
-                            nextPrintProgress += 10;
-                        }
-                    }
-                    logger.info("download file " + file.getName() + " end!");
-                    fos.flush();
-                } finally {
-                    IOUtils.closeQuietly(fos);
-                }
-                file.renameTo(new File(parentFile, fileName));
-            }
-        } finally {
-            IOUtils.closeQuietly(fos);
-        }
-    }
-
-    public void execute(Runnable runnable) throws InterruptedException {
-        taskQueue.put(runnable);
-    }
-
-    private class DownloadThread implements Runnable {
-
-        @Override
-        public void run() {
-            while (running) {
-                BinlogFile binlogFile = null;
-                try {
-                    binlogFile = downloadQueue.poll(5000, TimeUnit.MILLISECONDS);
-                    if (binlogFile != null) {
-                        int retry = 1;
-                        while (true) {
-                            try {
-                                download(binlogFile);
-                                break;
-                            } catch (Throwable e) {
-                                if (retry % 10 == 0) {
-                                    retry = retry + 1;
-                                    try {
-                                        logger.warn("download failed + " + binlogFile.toString() + "], retry : "
-                                                    + retry, e);
-                                        // File errorFile = new File(destDir,
-                                        // "error.txt");
-                                        // FileWriter writer = new
-                                        // FileWriter(errorFile);
-                                        // writer.write(ExceptionUtils.getFullStackTrace(e));
-                                        // writer.flush();
-                                        // IOUtils.closeQuietly(writer);
-                                        LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100 * retry));
-                                    } catch (Throwable e1) {
-                                        logger.error("write error failed", e1);
-                                    }
-                                } else {
-                                    retry = retry + 1;
-                                }
-                            }
-                        }
-                    }
-
-                    Runnable runnable = taskQueue.poll(5000, TimeUnit.MILLISECONDS);
-                    if (runnable != null) {
-                        runnable.run();
-                    }
-                } catch (Throwable e) {
-                    logger.error("task process failed", e);
-                }
-            }
-
-        }
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java

@@ -155,7 +155,6 @@ public class HttpHelper {
                     .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                     .build()))
                 .build();
-
             // ---------------- 创建支持https 的client成功---------
 
             URI uri = new URIBuilder(url).build();

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java

@@ -1,166 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
-import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
-import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-
-/**
- * aliyun rds的binlog parser支持
- * 
- * <pre>
- * 注意点：aliyun的binlog会有定期清理并备份到oss上, 这里实现了一份自动下载oss+rds binlog的机制
- * </pre>
- * 
- * @author chengjin.lyf on 2018/7/20 上午10:52
- * @since 1.0.25
- */
-public class RdsBinlogEventParserProxy extends MysqlEventParser {
-
-    private String                    rdsOpenApiUrl             = "https://rds.aliyuncs.com/";    // openapi地址
-    private String                    accesskey;                                                  // 云账号的ak
-    private String                    secretkey;                                                  // 云账号sk
-    private String                    instanceId;                                                 // rds实例id
-    private String                    directory;                                                  // binlog目录
-    private int                       batchFileSize             = 4;                              // 最多下载的binlog文件数量
-
-    private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = new RdsLocalBinlogEventParser();
-    private ExecutorService           executorService           = Executors.newSingleThreadExecutor(new ThreadFactory() {
-
-                                                                    @Override
-                                                                    public Thread newThread(Runnable r) {
-                                                                        Thread t = new Thread(r,
-                                                                            "rds-binlog-daemon-thread");
-                                                                        t.setDaemon(true);
-                                                                        return t;
-                                                                    }
-                                                                });
-
-    @Override
-    public void start() {
-        if (StringUtils.isNotEmpty(accesskey) && StringUtils.isNotEmpty(secretkey)
-            && StringUtils.isNotEmpty(instanceId)) {
-            final ParserExceptionHandler targetHandler = this.getParserExceptionHandler();
-            if (directory == null) {
-                directory = System.getProperty("java.io.tmpdir", "/tmp") + "/" + destination;
-            }
-            rdsLocalBinlogEventParser.setLogPositionManager(this.getLogPositionManager());
-            rdsLocalBinlogEventParser.setDestination(destination);
-            rdsLocalBinlogEventParser.setAlarmHandler(this.getAlarmHandler());
-            rdsLocalBinlogEventParser.setConnectionCharset(this.connectionCharset);
-            rdsLocalBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber);
-            rdsLocalBinlogEventParser.setEnableTsdb(this.enableTsdb);
-            rdsLocalBinlogEventParser.setEventBlackFilter(this.eventBlackFilter);
-            rdsLocalBinlogEventParser.setFilterQueryDcl(this.filterQueryDcl);
-            rdsLocalBinlogEventParser.setFilterQueryDdl(this.filterQueryDdl);
-            rdsLocalBinlogEventParser.setFilterQueryDml(this.filterQueryDml);
-            rdsLocalBinlogEventParser.setFilterRows(this.filterRows);
-            rdsLocalBinlogEventParser.setFilterTableError(this.filterTableError);
-            // rdsLocalBinlogEventParser.setIsGTIDMode(this.isGTIDMode);
-            rdsLocalBinlogEventParser.setMasterInfo(this.masterInfo);
-            rdsLocalBinlogEventParser.setEventFilter(this.eventFilter);
-            rdsLocalBinlogEventParser.setMasterPosition(this.masterPosition);
-            rdsLocalBinlogEventParser.setTransactionSize(this.transactionSize);
-            rdsLocalBinlogEventParser.setUrl(this.rdsOpenApiUrl);
-            rdsLocalBinlogEventParser.setAccesskey(this.accesskey);
-            rdsLocalBinlogEventParser.setSecretkey(this.secretkey);
-            rdsLocalBinlogEventParser.setInstanceId(this.instanceId);
-            rdsLocalBinlogEventParser.setEventSink(eventSink);
-            rdsLocalBinlogEventParser.setDirectory(directory);
-            rdsLocalBinlogEventParser.setBatchFileSize(batchFileSize);
-            rdsLocalBinlogEventParser.setParallel(this.parallel);
-            rdsLocalBinlogEventParser.setParallelBufferSize(this.parallelBufferSize);
-            rdsLocalBinlogEventParser.setParallelThreadSize(this.parallelThreadSize);
-            rdsLocalBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() {
-
-                @Override
-                public void onFinish() {
-                    executorService.execute(new Runnable() {
-
-                        @Override
-                        public void run() {
-                            rdsLocalBinlogEventParser.stop();
-                            RdsBinlogEventParserProxy.this.start();
-                        }
-                    });
-
-                }
-            });
-            this.setParserExceptionHandler(new ParserExceptionHandler() {
-
-                @Override
-                public void handle(Throwable e) {
-                    handleMysqlParserException(e);
-                    if (targetHandler != null) {
-                        targetHandler.handle(e);
-                    }
-                }
-            });
-        }
-
-        super.start();
-    }
-
-    private void handleMysqlParserException(Throwable throwable) {
-        if (throwable instanceof PositionNotFoundException) {
-            logger.info("remove rds not found position, try download rds binlog!");
-            executorService.execute(new Runnable() {
-
-                @Override
-                public void run() {
-                    try {
-                        logger.info("stop mysql parser!");
-                        RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
-                        long serverId = rdsBinlogEventParserProxy.getServerId();
-                        rdsLocalBinlogEventParser.setServerId(serverId);
-                        rdsBinlogEventParserProxy.stop();
-                        logger.info("start rds mysql binlog parser!");
-                        rdsLocalBinlogEventParser.start();
-                    } catch (Throwable e) {
-                        logger.info("handle exception failed", e);
-                    }
-                }
-            });
-        }
-    }
-
-    @Override
-    public void stop() {
-        super.stop();
-    }
-
-    @Override
-    public boolean isStart() {
-        return super.isStart();
-    }
-
-    public void setRdsOpenApiUrl(String rdsOpenApiUrl) {
-        this.rdsOpenApiUrl = rdsOpenApiUrl;
-    }
-
-    public void setAccesskey(String accesskey) {
-        this.accesskey = accesskey;
-    }
-
-    public void setSecretkey(String secretkey) {
-        this.secretkey = secretkey;
-    }
-
-    public void setInstanceId(String instanceId) {
-        this.instanceId = instanceId;
-    }
-
-    public void setDirectory(String directory) {
-        this.directory = directory;
-    }
-
-    public void setBatchFileSize(int batchFileSize) {
-        this.batchFileSize = batchFileSize;
-    }
-
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java

@@ -1,21 +1,46 @@
 package com.alibaba.otter.canal.parse.inbound.mysql.rds;
 
-import java.net.URI;
+import io.netty.handler.codec.http.HttpResponseStatus;
-import java.net.URISyntaxException;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.text.SimpleDateFormat;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.UUID;
 
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
+import com.alibaba.fastjson.JSON;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
+import com.alibaba.fastjson.JSONArray;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
+import com.alibaba.fastjson.JSONObject;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsItem;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBackupPolicyRequest;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFilesRequest;
 
 /**
  * @author agapple 2017年10月14日 下午1:53:52
@@ -23,78 +48,287 @@ import com.alibaba.otter.canal.parse.inbound.mysql.rds.request.DescribeBinlogFil
  */
 public class RdsBinlogOpenApi {
 
-    protected static final Logger logger = LoggerFactory.getLogger(RdsBinlogOpenApi.class);
+    protected static final Logger logger              = LoggerFactory.getLogger(RdsBinlogOpenApi.class);
+    private static final String   ISO8601_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+    private static final int      TIMEOUT             = 10000;
+    private static final String   ENCODING            = "UTF-8";
+    private static final String   MAC_NAME            = "HmacSHA1";
+    private static final String   API_VERSION         = "2014-08-15";
+    private static final String   SIGNATURE_VERSION   = "1.0";
 
-    public static List<BinlogFile> listBinlogFiles(String url, String ak, String sk, String dbInstanceId,
+    public static void downloadBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
-                                                   Date startTime, Date endTime) {
+                                           Date endTime, File destDir) throws Throwable {
-        DescribeBinlogFilesRequest request = new DescribeBinlogFilesRequest();
+        int pageSize = 100;
-        if (StringUtils.isNotEmpty(url)) {
+        int pageNumber = 0;
-            try {
+        int pageRecordCount = 1;
-                URI uri = new URI(url);
+        String hostInstanceID = null;
-                request.setEndPoint(uri.getHost());
+        while (pageRecordCount > 0 && pageRecordCount <= pageSize) {
-            } catch (URISyntaxException e) {
+            pageNumber += 1;
-                logger.error("resolve url host failed, will use default rds endpoint!");
+            String result = describeBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime, pageSize, pageNumber);
-            }
+            JSONObject jsobObj = JSON.parseObject(result);
-        }
+            pageRecordCount = jsobObj.getInteger("PageRecordCount");
-        request.setStartDate(startTime);
+
-        request.setEndDate(endTime);
+            if (pageRecordCount > 0) {
-        request.setPageNumber(1);
+                FileUtils.forceMkdir(destDir);
-        request.setPageSize(100);
+                File hostIdFile = new File(destDir, "hostId");
-        request.setRdsInstanceId(dbInstanceId);
+                if (hostIdFile.exists()) {
-        request.setAccessKeyId(ak);
+                    List<String> lines = IOUtils.readLines(new FileInputStream(hostIdFile));
-        request.setAccessKeySecret(sk);
+                    hostInstanceID = StringUtils.join(lines, "\n");
-        DescribeBinlogFileResult result = null;
-        int retryTime = 3;
-        while (true) {
-            try {
-                result = request.doAction();
-                break;
-            } catch (Exception e) {
-                if (retryTime-- <= 0) {
-                    throw new RuntimeException(e);
                 }
-                try {
+
-                    Thread.sleep(100L);
+                String itemStr = jsobObj.getString("Items");
-                } catch (InterruptedException e1) {
+                JSONObject binLogFileObj = JSONObject.parseObject(itemStr);
+                JSONArray items = binLogFileObj.getJSONArray("BinLogFile");
+                if (items == null || items.isEmpty()) {
+                    continue;
+                }
+                for (int i = 0; i < items.size(); i++) {
+                    JSONObject item = (JSONObject) items.get(i);
+                    String oneHostInstanceID = item.getString("HostInstanceID");
+                    if (hostInstanceID == null) {
+                        hostInstanceID = oneHostInstanceID;
+                        FileOutputStream hostIdFileOut = null;
+                        try {
+                            hostIdFileOut = new FileOutputStream(hostIdFile);
+                            hostIdFileOut.write(oneHostInstanceID.getBytes());
+                            hostIdFileOut.flush();
+                        } finally {
+                            IOUtils.closeQuietly(hostIdFileOut);
+                        }
+                    }
+
+                    if (hostInstanceID.equals(oneHostInstanceID)) { // 只选择一个host下载
+                        String downloadLink = item.getString("DownloadLink");
+                        String fileName = StringUtils.substringBetween(downloadLink, "mysql-bin.", ".tar");
+                        if (StringUtils.isNotEmpty(fileName)) {
+                            File currentFile = new File(destDir, "mysql-bin." + fileName);
+                            if (currentFile.isFile() && currentFile.exists()) {
+                                // 检查一下文件是否存在，存在就就没必要下载了
+                                continue;
+                            }
+                        }
+
+                        HttpGet httpGet = new HttpGet(downloadLink);
+                        CloseableHttpClient httpClient = HttpClientBuilder.create()
+                            .setMaxConnPerRoute(50)
+                            .setMaxConnTotal(100)
+                            .build();
+                        RequestConfig requestConfig = RequestConfig.custom()
+                            .setConnectTimeout(TIMEOUT)
+                            .setConnectionRequestTimeout(TIMEOUT)
+                            .setSocketTimeout(TIMEOUT)
+                            .build();
+                        httpGet.setConfig(requestConfig);
+                        HttpResponse response = httpClient.execute(httpGet);
+                        int statusCode = response.getStatusLine().getStatusCode();
+                        if (statusCode != HttpResponseStatus.OK.code()) {
+                            throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:"
+                                                       + statusCode);
+                        }
+                        saveFile(destDir, response);
+                    }
                 }
             }
         }
-        if (result == null) {
-            return Collections.EMPTY_LIST;
-        }
-        RdsItem rdsItem = result.getItems();
-        if (rdsItem != null) {
-            return rdsItem.getBinLogFile();
-        }
-        return Collections.EMPTY_LIST;
     }
 
-    public static RdsBackupPolicy queryBinlogBackupPolicy(String url, String ak, String sk, String dbInstanceId) {
+    private static void saveFile(File parentFile, HttpResponse response) throws IOException {
-        DescribeBackupPolicyRequest request = new DescribeBackupPolicyRequest();
+        InputStream is = response.getEntity().getContent();
-        if (StringUtils.isNotEmpty(url)) {
+        long totalSize = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
-            try {
+        String fileName = response.getFirstHeader("Content-Disposition").getValue();
-                URI uri = new URI(url);
+        fileName = StringUtils.substringAfter(fileName, "filename=");
-                request.setEndPoint(uri.getHost());
+        boolean isTar = StringUtils.endsWith(fileName, ".tar");
-            } catch (URISyntaxException e) {
+        FileUtils.forceMkdir(parentFile);
-                logger.error("resolve url host failed, will use default rds endpoint!");
+        FileOutputStream fos = null;
-            }
+        try {
-        }
+            if (isTar) {
-        request.setRdsInstanceId(dbInstanceId);
+                TarArchiveInputStream tais = new TarArchiveInputStream(is);
-        request.setAccessKeyId(ak);
+                TarArchiveEntry tarArchiveEntry = null;
-        request.setAccessKeySecret(sk);
+                while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
-        int retryTime = 3;
+                    String name = tarArchiveEntry.getName();
-        while (true) {
+                    File tarFile = new File(parentFile, name);
-            try {
+                    logger.info("start to download file " + tarFile.getName());
-                return request.doAction();
+                    BufferedOutputStream bos = null;
-            } catch (Exception e) {
+                    try {
-                if (retryTime-- <= 0) {
+                        bos = new BufferedOutputStream(new FileOutputStream(tarFile));
-                    throw new RuntimeException(e);
+                        int read = -1;
+                        byte[] buffer = new byte[1024];
+                        while ((read = tais.read(buffer)) != -1) {
+                            bos.write(buffer, 0, read);
+                        }
+                        logger.info("download file " + tarFile.getName() + " end!");
+                    } finally {
+                        IOUtils.closeQuietly(bos);
+                    }
+                }
+                tais.close();
+            } else {
+                File file = new File(parentFile, fileName);
+                if (!file.isFile()) {
+                    file.createNewFile();
                 }
                 try {
-                    Thread.sleep(100L);
+                    fos = new FileOutputStream(file);
-                } catch (InterruptedException e1) {
+                    byte[] buffer = new byte[1024];
+                    int len;
+                    long copySize = 0;
+                    long nextPrintProgress = 0;
+                    logger.info("start to download file " + file.getName());
+                    while ((len = is.read(buffer)) != -1) {
+                        fos.write(buffer, 0, len);
+                        copySize += len;
+                        long progress = copySize * 100 / totalSize;
+                        if (progress >= nextPrintProgress) {
+                            logger.info("download " + file.getName() + " progress : " + progress
+                                        + "% , download size : " + copySize + ", total size : " + totalSize);
+                            nextPrintProgress += 10;
+                        }
+                    }
+                    logger.info("download file " + file.getName() + " end!");
+                    fos.flush();
+                } finally {
+                    IOUtils.closeQuietly(fos);
                 }
             }
+        } finally {
+            IOUtils.closeQuietly(fos);
         }
     }
+
+    public static String describeBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime,
+                                             Date endTime, int pageSize, int pageNumber) throws Exception {
+        Map<String, String> paramMap = new HashMap<String, String>();
+        paramMap.put("Action", "DescribeBinlogFiles");
+        paramMap.put("DBInstanceId", dbInstanceId); // rds实例id
+        paramMap.put("StartTime", formatIso8601Date(startTime));
+        paramMap.put("EndTime", formatIso8601Date(endTime));
+        paramMap.put("PageSize", String.valueOf(pageSize));
+        paramMap.put("PageNumber", String.valueOf(pageNumber));
+        return doRequest(url, paramMap, ak, sk);
+    }
+
+    private static String doRequest(String domin, Map<String, String> param, String ak, String sk) throws Exception {
+        param.put("AccessKeyId", ak);
+        param.put("SignatureMethod", "HMAC-SHA1");
+        param.put("SignatureVersion", SIGNATURE_VERSION);
+        param.put("Version", API_VERSION);
+        param.put("SignatureNonce", UUID.randomUUID().toString());
+        param.put("Format", "JSON");
+        param.put("Timestamp", formatIso8601Date(new Date()));
+        String signStr = generate("POST", param, sk);
+        param.put("Signature", signStr);
+        String request = concatQueryString(param);
+        String url = domin + "?" + request;
+        String result = HttpHelper.post(url, null, Collections.EMPTY_MAP, TIMEOUT);
+        return result;
+    }
+
+    public static String concatQueryString(Map<String, String> parameters) throws UnsupportedEncodingException {
+        if (null == parameters) {
+            return null;
+        }
+        StringBuilder urlBuilder = new StringBuilder("");
+        for (Map.Entry<String, String> entry : parameters.entrySet()) {
+            String key = entry.getKey();
+            String val = entry.getValue();
+            urlBuilder.append(encode(key));
+            if (val != null) {
+                urlBuilder.append("=").append(encode(val));
+            }
+            urlBuilder.append("&");
+        }
+        int strIndex = urlBuilder.length();
+        if (parameters.size() > 0) {
+            urlBuilder.deleteCharAt(strIndex - 1);
+        }
+        return urlBuilder.toString();
+    }
+
+    public static String encode(String value) throws UnsupportedEncodingException {
+        return URLEncoder.encode(value, "UTF-8");
+    }
+
+    private static String formatIso8601Date(Date date) {
+        SimpleDateFormat df = new SimpleDateFormat(ISO8601_DATE_FORMAT);
+        df.setTimeZone(TimeZone.getTimeZone("GMT"));
+        return df.format(date);
+    }
+
+    /**
+     * 使用 HMAC-SHA1 签名方法对对encryptText进行签名
+     *
+     * @param encryptText 被签名的字符串
+     * @param encryptKey 密钥
+     * @return
+     * @throws Exception
+     */
+    public static byte[] HmacSHA1Encrypt(String encryptText, String encryptKey) throws Exception {
+        byte[] data = encryptKey.getBytes(ENCODING);
+        // 根据给定的字节数组构造一个密钥,第二参数指定一个密钥算法的名称
+        SecretKey secretKey = new SecretKeySpec(data, MAC_NAME);
+        // 生成一个指定 Mac 算法 的 Mac 对象
+        Mac mac = Mac.getInstance(MAC_NAME);
+        // 用给定密钥初始化 Mac 对象
+        mac.init(secretKey);
+
+        byte[] text = encryptText.getBytes(ENCODING);
+        // 完成 Mac 操作
+        return mac.doFinal(text);
+    }
+
+    private static String base64(byte input[]) throws UnsupportedEncodingException {
+        return new String(Base64.encodeBase64(input), ENCODING);
+    }
+
+    /** 对参数名称和参数值进行URL编码 **/
+    public static String generate(String method, Map<String, String> parameter, String accessKeySecret)
+                                                                                                       throws Exception {
+        String signString = generateSignString(method, parameter);
+        byte[] signBytes = HmacSHA1Encrypt(signString, accessKeySecret + "&");
+        String signature = base64(signBytes);
+        if ("POST".equals(method)) {
+            return signature;
+        }
+        return URLEncoder.encode(signature, "UTF-8");
+    }
+
+    private static String generateQueryString(TreeMap<String, String> treeMap) {
+        StringBuilder canonicalizedQueryString = new StringBuilder();
+        boolean first = true;
+        for (String key : treeMap.navigableKeySet()) {
+            String value = treeMap.get(key);
+            if (!first) {
+                canonicalizedQueryString.append("&");
+            }
+            first = false;
+            canonicalizedQueryString.append(percentEncode(key)).append("=").append(percentEncode(value));
+        }
+        return canonicalizedQueryString.toString();
+    }
+
+    public static String generateSignString(String httpMethod, Map<String, String> parameter) throws IOException {
+        TreeMap<String, String> sortParameter = new TreeMap<String, String>();
+        sortParameter.putAll(parameter);
+        String canonicalizedQueryString = generateQueryString(sortParameter);
+        if (null == httpMethod) {
+            throw new RuntimeException("httpMethod can not be empty");
+        }
+        /** 构造待签名的字符串* */
+        StringBuilder stringToSign = new StringBuilder();
+        stringToSign.append(httpMethod).append("&");
+        stringToSign.append(percentEncode("/")).append("&");
+        stringToSign.append(percentEncode(canonicalizedQueryString));
+        return stringToSign.toString();
+    }
+
+    public static String percentEncode(String value) {
+        try {
+            return value == null ? null : URLEncoder.encode(value, ENCODING)
+                .replaceAll("\\+", "%20")
+                .replaceAll("\\*", "%2A")
+                .replaceAll("%7E", "~");
+        } catch (Exception e) {
+        }
+        return "";
+    }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java

@@ -2,23 +2,14 @@ package com.alibaba.otter.canal.parse.inbound.mysql.rds;
 
 import java.io.File;
 import java.util.Date;
-import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
 import org.springframework.util.Assert;
 
 import com.alibaba.otter.canal.parse.CanalEventParser;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
-import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
-import com.alibaba.otter.canal.parse.exception.ServerIdNotMatchException;
-import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
-import com.alibaba.otter.canal.parse.inbound.ParserExceptionHandler;
-import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinLogConnection;
 import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
-import com.alibaba.otter.canal.protocol.position.LogPosition;
 
 /**
  * 基于rds binlog备份文件的复制
@@ -26,104 +17,47 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
  * @author agapple 2017年10月15日 下午1:27:36
  * @since 1.0.25
  */
-public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser, LocalBinLogConnection.FileParserListener {
+public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser {
 
-    private String              url;                // openapi地址
+    private String url = "https://rds.aliyuncs.com/"; // openapi地址
-    private String              accesskey;          // 云账号的ak
+    private String accesskey;                        // 云账号的ak
-    private String              secretkey;          // 云账号sk
+    private String secretkey;                        // 云账号sk
-    private String              instanceId;         // rds实例id
+    private String instanceId;                       // rds实例id
-    private Long                startTime;
+    private Long   startTime;
-    private Long                endTime;
+    private Long   endTime;
-    private BinlogDownloadQueue binlogDownloadQueue;
-    private ParseFinishListener finishListener;
-    private int                 batchFileSize;
 
     public RdsLocalBinlogEventParser(){
     }
 
     public void start() throws CanalParseException {
         try {
+            Assert.notNull(startTime);
             Assert.notNull(accesskey);
             Assert.notNull(secretkey);
             Assert.notNull(instanceId);
             Assert.notNull(url);
-            Assert.notNull(directory);
-
             if (endTime == null) {
                 endTime = System.currentTimeMillis();
             }
 
-            EntryPosition entryPosition = findStartPosition(null);
+            RdsBinlogOpenApi.downloadBinlogFiles(url,
-            if (entryPosition == null) {
-                throw new PositionNotFoundException("position not found!");
-            }
-            long startTimeInMill = entryPosition.getTimestamp();
-            startTime = startTimeInMill;
-            List<BinlogFile> binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url,
                 accesskey,
                 secretkey,
                 instanceId,
                 new Date(startTime),
-                new Date(endTime));
+                new Date(endTime),
-            binlogDownloadQueue = new BinlogDownloadQueue(binlogFiles, batchFileSize, directory);
+                new File(directory));
-            binlogDownloadQueue.silenceDownload();
+
-            needWait = true;
+            // 更新一下时间戳
-            // try to download one file,use to test server id
+            masterPosition = new EntryPosition(startTime);
-            binlogDownloadQueue.tryOne();
         } catch (Throwable e) {
             logger.error("download binlog failed", e);
             throw new CanalParseException(e);
         }
-        setParserExceptionHandler(new ParserExceptionHandler() {
 
-            @Override
-            public void handle(Throwable e) {
-                handleMysqlParserException(e);
-            }
-        });
         super.start();
     }
 
-    private void handleMysqlParserException(Throwable throwable) {
-        if (throwable instanceof ServerIdNotMatchException) {
-            logger.error("server id not match, try download another rds binlog!");
-            binlogDownloadQueue.notifyNotMatch();
-            try {
-                binlogDownloadQueue.cleanDir();
-                binlogDownloadQueue.tryOne();
-                binlogDownloadQueue.prepare();
-            } catch (Throwable e) {
-                throw new RuntimeException(e);
-            }
-
-            try {
-                binlogDownloadQueue.execute(new Runnable() {
-
-                    @Override
-                    public void run() {
-                        RdsLocalBinlogEventParser.super.stop();
-                        RdsLocalBinlogEventParser.super.start();
-                    }
-                });
-            } catch (InterruptedException e) {
-                throw new RuntimeException(e);
-            }
-
-        }
-    }
-
-    @Override
-    protected ErosaConnection buildErosaConnection() {
-        ErosaConnection connection = super.buildErosaConnection();
-        if (connection instanceof LocalBinLogConnection) {
-            LocalBinLogConnection localBinLogConnection = (LocalBinLogConnection) connection;
-            localBinLogConnection.setNeedWait(true);
-            localBinLogConnection.setServerId(serverId);
-            localBinLogConnection.setParserListener(this);
-        }
-        return connection;
-    }
-
     public String getUrl() {
         return url;
     }
@@ -134,87 +68,44 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
         }
     }
 
+    public String getAccesskey() {
+        return accesskey;
+    }
+
     public void setAccesskey(String accesskey) {
         this.accesskey = accesskey;
     }
 
+    public String getSecretkey() {
+        return secretkey;
+    }
+
     public void setSecretkey(String secretkey) {
         this.secretkey = secretkey;
     }
 
+    public String getInstanceId() {
+        return instanceId;
+    }
+
     public void setInstanceId(String instanceId) {
         this.instanceId = instanceId;
     }
 
+    public Long getStartTime() {
+        return startTime;
+    }
+
     public void setStartTime(Long startTime) {
         this.startTime = startTime;
     }
 
+    public Long getEndTime() {
+        return endTime;
+    }
+
     public void setEndTime(Long endTime) {
         this.endTime = endTime;
     }
 
-    @Override
-    public void onFinish(String fileName) {
-        try {
-            binlogDownloadQueue.downOne();
-            File needDeleteFile = new File(directory + File.separator + fileName);
-            if (needDeleteFile.exists()) {
-                needDeleteFile.delete();
-            }
-            // 处理下logManager位点问题
-            LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
-            Long timestamp = 0L;
-            if (logPosition != null && logPosition.getPostion() != null) {
-                timestamp = logPosition.getPostion().getTimestamp();
-                EntryPosition position = logPosition.getPostion();
-                LogPosition newLogPosition = new LogPosition();
-                String journalName = position.getJournalName();
-                int sepIdx = journalName.indexOf(".");
-                String fileIndex = journalName.substring(sepIdx + 1);
-                int index = NumberUtils.toInt(fileIndex) + 1;
-                String newJournalName = journalName.substring(0, sepIdx) + "."
-                                        + StringUtils.leftPad(String.valueOf(index), fileIndex.length(), "0");
-                newLogPosition.setPostion(new EntryPosition(newJournalName,
-                    4L,
-                    position.getTimestamp(),
-                    position.getServerId()));
-                newLogPosition.setIdentity(logPosition.getIdentity());
-                logPositionManager.persistLogPosition(destination, newLogPosition);
-            }
-
-            if (binlogDownloadQueue.isLastFile(fileName)) {
-                logger.warn("last file : " + fileName + " , timestamp : " + timestamp
-                            + " , all file parse complete, switch to mysql parser!");
-                finishListener.onFinish();
-                return;
-            } else {
-                logger.warn("parse local binlog file : " + fileName + " , timestamp : " + timestamp
-                            + " , try the next binlog !");
-            }
-            binlogDownloadQueue.prepare();
-        } catch (Exception e) {
-            logger.error("prepare download binlog file failed!", e);
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public void stop() {
-        this.binlogDownloadQueue.release();
-        super.stop();
-    }
-
-    public void setFinishListener(ParseFinishListener finishListener) {
-        this.finishListener = finishListener;
-    }
-
-    public interface ParseFinishListener {
-
-        void onFinish();
-    }
-
-    public void setBatchFileSize(int batchFileSize) {
-        this.batchFileSize = batchFileSize;
-    }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/BinlogFile.java

@@ -1,80 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class BinlogFile {
-
-    private Long   FileSize;
-    private String LogBeginTime;
-    private String LogEndTime;
-    private String DownloadLink;
-    private String HostInstanceID;
-    private String LinkExpiredTime;
-    private String fileName;
-
-    public Long getFileSize() {
-        return FileSize;
-    }
-
-    public void setFileSize(Long fileSize) {
-        FileSize = fileSize;
-    }
-
-    public String getLogBeginTime() {
-        return LogBeginTime;
-    }
-
-    public void setLogBeginTime(String logBeginTime) {
-        LogBeginTime = logBeginTime;
-    }
-
-    public String getLogEndTime() {
-        return LogEndTime;
-    }
-
-    public void setLogEndTime(String logEndTime) {
-        LogEndTime = logEndTime;
-    }
-
-    public String getDownloadLink() {
-        return DownloadLink;
-    }
-
-    public void setDownloadLink(String downloadLink) {
-        DownloadLink = downloadLink;
-    }
-
-    public String getHostInstanceID() {
-        return HostInstanceID;
-    }
-
-    public void setHostInstanceID(String hostInstanceID) {
-        HostInstanceID = hostInstanceID;
-    }
-
-    public String getLinkExpiredTime() {
-        return LinkExpiredTime;
-    }
-
-    public void setLinkExpiredTime(String linkExpiredTime) {
-        LinkExpiredTime = linkExpiredTime;
-    }
-
-    public String getFileName() {
-        return fileName;
-    }
-
-    public void setFileName(String fileName) {
-        this.fileName = fileName;
-    }
-
-    @Override
-    public String toString() {
-        return "BinlogFile [FileSize=" + FileSize + ", LogBeginTime=" + LogBeginTime + ", LogEndTime=" + LogEndTime
-               + ", DownloadLink=" + DownloadLink + ", HostInstanceID=" + HostInstanceID + ", LinkExpiredTime="
-               + LinkExpiredTime + ", fileName=" + fileName + "]";
-    }
-
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/DescribeBinlogFileResult.java

@@ -1,63 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class DescribeBinlogFileResult {
-
-    private RdsItem Items;
-    private long    PageNumber;
-    private long    TotalRecordCount;
-    private long    TotalFileSize;
-    private String  RequestId;
-    private long    PageRecordCount;
-
-    public RdsItem getItems() {
-        return Items;
-    }
-
-    public void setItems(RdsItem items) {
-        Items = items;
-    }
-
-    public long getPageNumber() {
-        return PageNumber;
-    }
-
-    public void setPageNumber(long pageNumber) {
-        PageNumber = pageNumber;
-    }
-
-    public long getTotalRecordCount() {
-        return TotalRecordCount;
-    }
-
-    public void setTotalRecordCount(long totalRecordCount) {
-        TotalRecordCount = totalRecordCount;
-    }
-
-    public long getTotalFileSize() {
-        return TotalFileSize;
-    }
-
-    public void setTotalFileSize(long totalFileSize) {
-        TotalFileSize = totalFileSize;
-    }
-
-    public String getRequestId() {
-        return RequestId;
-    }
-
-    public void setRequestId(String requestId) {
-        RequestId = requestId;
-    }
-
-    public long getPageRecordCount() {
-        return PageRecordCount;
-    }
-
-    public void setPageRecordCount(long pageRecordCount) {
-        PageRecordCount = pageRecordCount;
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsBackupPolicy.java

@@ -1,78 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class RdsBackupPolicy {
-
-    /**
-     * 数据备份保留天数（7到730天）。
-     */
-    private String  BackupRetentionPeriod;
-    /**
-     * 数据备份时间，格式：HH:mmZ- HH:mm Z。
-     */
-    private String  PreferredBackupTime;
-    /**
-     * 数据备份周期。Monday：周一；Tuesday：周二；Wednesday：周三；Thursday：周四；Friday：周五；Saturday：
-     * 周六；Sunday：周日。
-     */
-    private String  PreferredBackupPeriod;
-    /**
-     * 日志备份状态。Enable：开启；Disabled：关闭。
-     */
-    private boolean BackupLog;
-    /**
-     * 日志备份保留天数（7到730天）。
-     */
-    private int     LogBackupRetentionPeriod;
-
-    public String getBackupRetentionPeriod() {
-        return BackupRetentionPeriod;
-    }
-
-    public void setBackupRetentionPeriod(String backupRetentionPeriod) {
-        BackupRetentionPeriod = backupRetentionPeriod;
-    }
-
-    public String getPreferredBackupTime() {
-        return PreferredBackupTime;
-    }
-
-    public void setPreferredBackupTime(String preferredBackupTime) {
-        PreferredBackupTime = preferredBackupTime;
-    }
-
-    public String getPreferredBackupPeriod() {
-        return PreferredBackupPeriod;
-    }
-
-    public void setPreferredBackupPeriod(String preferredBackupPeriod) {
-        PreferredBackupPeriod = preferredBackupPeriod;
-    }
-
-    public boolean isBackupLog() {
-        return BackupLog;
-    }
-
-    public void setBackupLog(boolean backupLog) {
-        BackupLog = backupLog;
-    }
-
-    public int getLogBackupRetentionPeriod() {
-        return LogBackupRetentionPeriod;
-    }
-
-    public void setLogBackupRetentionPeriod(int logBackupRetentionPeriod) {
-        LogBackupRetentionPeriod = logBackupRetentionPeriod;
-    }
-
-    @Override
-    public String toString() {
-        return "RdsBackupPolicy [BackupRetentionPeriod=" + BackupRetentionPeriod + ", PreferredBackupTime="
-               + PreferredBackupTime + ", PreferredBackupPeriod=" + PreferredBackupPeriod + ", BackupLog=" + BackupLog
-               + ", LogBackupRetentionPeriod=" + LogBackupRetentionPeriod + "]";
-    }
-
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java

@@ -1,26 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.data;
-
-import java.util.List;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public class RdsItem {
-
-    private List<BinlogFile> BinLogFile;
-
-    public List<BinlogFile> getBinLogFile() {
-        return BinLogFile;
-    }
-
-    public void setBinLogFile(List<BinlogFile> binLogFile) {
-        BinLogFile = binLogFile;
-    }
-
-    @Override
-    public String toString() {
-        return "RdsItem [BinLogFile=" + BinLogFile + "]";
-    }
-
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java

@@ -1,251 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
-
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import javax.net.ssl.SSLContext;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.HttpClientConnectionManager;
-import org.apache.http.conn.socket.PlainConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.TrustStrategy;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.ssl.SSLContexts;
-import org.apache.http.util.EntityUtils;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午2:26
- * @since 1.0.25
- */
-public abstract class AbstractRequest<T> {
-
-    /**
-     * 要求的编码格式
-     */
-    private static final String ENCODING = "UTF-8";
-    /**
-     * 要求的sign签名算法
-     */
-    private static final String MAC_NAME = "HmacSHA1";
-
-    private String              accessKeyId;
-
-    private String              accessKeySecret;
-
-    /**
-     * api 版本
-     */
-    private String              version;
-
-    private String              endPoint = "rds.aliyuncs.com";
-
-    private String              protocol = "http";
-
-    public void setProtocol(String protocol) {
-        this.protocol = protocol;
-    }
-
-    private int                 timeout = (int) TimeUnit.MINUTES.toMillis(1);
-
-    private Map<String, String> treeMap = new TreeMap();
-
-    public void putQueryString(String name, String value) {
-        if (StringUtils.isBlank(name) || StringUtils.isBlank(value)) {
-            return;
-        }
-        treeMap.put(name, value);
-    }
-
-    public void setVersion(String version) {
-        this.version = version;
-    }
-
-    public void setEndPoint(String endPoint) {
-        this.endPoint = endPoint;
-    }
-
-    public void setAccessKeyId(String accessKeyId) {
-        this.accessKeyId = accessKeyId;
-    }
-
-    public void setAccessKeySecret(String accessKeySecret) {
-        this.accessKeySecret = accessKeySecret;
-    }
-
-    /**
-     * 使用 HMAC-SHA1 签名方法对对encryptText进行签名
-     *
-     * @param encryptText 被签名的字符串
-     * @param encryptKey 密钥
-     * @return
-     * @throws Exception
-     */
-    private byte[] HmacSHA1Encrypt(String encryptText, String encryptKey) throws Exception {
-        byte[] data = encryptKey.getBytes(ENCODING);
-        // 根据给定的字节数组构造一个密钥,第二参数指定一个密钥算法的名称
-        SecretKey secretKey = new SecretKeySpec(data, MAC_NAME);
-        // 生成一个指定 Mac 算法 的 Mac 对象
-        Mac mac = Mac.getInstance(MAC_NAME);
-        // 用给定密钥初始化 Mac 对象
-        mac.init(secretKey);
-
-        byte[] text = encryptText.getBytes(ENCODING);
-        // 完成 Mac 操作
-        return mac.doFinal(text);
-    }
-
-    private String base64(byte input[]) throws UnsupportedEncodingException {
-        return new String(Base64.encodeBase64(input), ENCODING);
-    }
-
-    private String concatQueryString(Map<String, String> parameters) throws UnsupportedEncodingException {
-        if (null == parameters) {
-            return null;
-        }
-        StringBuilder urlBuilder = new StringBuilder("");
-        for (Map.Entry<String, String> entry : parameters.entrySet()) {
-            String key = entry.getKey();
-            String val = entry.getValue();
-            urlBuilder.append(encode(key));
-            if (val != null) {
-                urlBuilder.append("=").append(encode(val));
-            }
-            urlBuilder.append("&");
-        }
-        int strIndex = urlBuilder.length();
-        if (parameters.size() > 0) {
-            urlBuilder.deleteCharAt(strIndex - 1);
-        }
-        return urlBuilder.toString();
-    }
-
-    private String encode(String value) throws UnsupportedEncodingException {
-        return URLEncoder.encode(value, "UTF-8");
-    }
-
-    private String makeSignature(TreeMap<String, String> paramMap) throws Exception {
-        String cqs = concatQueryString(paramMap);
-        cqs = encode(cqs);
-        cqs = cqs.replaceAll("\\+", "%20");
-        cqs = cqs.replaceAll("\\*", "%2A");
-        cqs = cqs.replaceAll("%7E", "~");
-        StringBuilder stringBuilder = new StringBuilder();
-        stringBuilder.append("GET").append("&").append(encode("/")).append("&").append(cqs);
-        return base64(HmacSHA1Encrypt(stringBuilder.toString(), accessKeySecret + "&"));
-    }
-
-    public final String formatUTCTZ(Date date) {
-        SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'");
-        sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
-        return sdf.format(date);
-    }
-
-    private void fillCommonParam(Map<String, String> p) {
-        p.put("Format", "JSON");
-        p.put("Version", version);
-        p.put("AccessKeyId", accessKeyId);
-        p.put("SignatureMethod", "HMAC-SHA1"); // 此处不能用变量 MAC_NAME
-        p.put("Timestamp", formatUTCTZ(new Date()));
-        p.put("SignatureVersion", "1.0");
-        p.put("SignatureNonce", UUID.randomUUID().toString());
-    }
-
-    private String makeRequestString(Map<String, String> param) throws Exception {
-        fillCommonParam(param);
-        String sign = makeSignature(new TreeMap<String, String>(param));
-        StringBuilder builder = new StringBuilder();
-        for (Map.Entry<String, String> entry : param.entrySet()) {
-            builder.append(encode(entry.getKey())).append("=").append(encode(entry.getValue())).append("&");
-        }
-        builder.append("Signature").append("=").append(sign);
-        return builder.toString();
-    }
-
-    /**
-     * 执行http请求
-     *
-     * @param getMethod
-     * @return
-     * @throws IOException
-     */
-    @SuppressWarnings("deprecation")
-    private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
-        SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
-
-            @Override
-            public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
-                return true;
-            }
-        }).build();
-        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
-            new String[] { "TLSv1" },
-            null,
-            SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
-        Registry registry = RegistryBuilder.create()
-            .register("http", PlainConnectionSocketFactory.INSTANCE)
-            .register("https", sslsf)
-            .build();
-        HttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager(registry);
-        CloseableHttpClient httpClient = HttpClientBuilder.create()
-            .setMaxConnPerRoute(50)
-            .setMaxConnTotal(100)
-            .setConnectionManager(httpClientConnectionManager)
-            .build();
-        RequestConfig requestConfig = RequestConfig.custom()
-            .setConnectTimeout(timeout)
-            .setConnectionRequestTimeout(timeout)
-            .setSocketTimeout(timeout)
-            .build();
-        getMethod.setConfig(requestConfig);
-        HttpResponse response = httpClient.execute(getMethod);
-        int statusCode = response.getStatusLine().getStatusCode();
-        if (statusCode != HttpResponseStatus.OK.code() && statusCode != HttpResponseStatus.PARTIAL_CONTENT.code()) {
-            String result = EntityUtils.toString(response.getEntity());
-            throw new RuntimeException("return error !" + response.getStatusLine().getReasonPhrase() + ", " + result);
-        }
-        return response;
-    }
-
-    protected abstract T processResult(HttpResponse response) throws Exception;
-
-    protected void processBefore() {
-
-    }
-
-    public final T doAction() throws Exception {
-        processBefore();
-        String requestStr = makeRequestString(treeMap);
-        HttpGet httpGet = new HttpGet(protocol + "://" + endPoint + "?" + requestStr);
-        HttpResponse response = executeHttpRequest(httpGet, endPoint);
-        if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
-            String result = EntityUtils.toString(response.getEntity());
-            throw new RuntimeException("http request failed! " + result);
-        }
-        return processResult(response);
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java

@@ -1,40 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.util.EntityUtils;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
-
-/**
- * rds 备份策略查询
- * 
- * @author chengjin.lyf on 2018/8/7 下午3:41
- * @since 1.0.25
- */
-public class DescribeBackupPolicyRequest extends AbstractRequest<RdsBackupPolicy> {
-
-    public DescribeBackupPolicyRequest(){
-        setVersion("2014-08-15");
-        putQueryString("Action", "DescribeBackupPolicy");
-
-    }
-
-    public void setRdsInstanceId(String rdsInstanceId) {
-        putQueryString("DBInstanceId", rdsInstanceId);
-    }
-
-    @Override
-    protected RdsBackupPolicy processResult(HttpResponse response) throws Exception {
-        String result = EntityUtils.toString(response.getEntity());
-        JSONObject jsonObj = JSON.parseObject(result);
-        RdsBackupPolicy policy = new RdsBackupPolicy();
-        policy.setBackupRetentionPeriod(jsonObj.getString("BackupRetentionPeriod"));
-        policy.setBackupLog(jsonObj.getString("BackupLog").equalsIgnoreCase("Enable"));
-        policy.setLogBackupRetentionPeriod(jsonObj.getIntValue("LogBackupRetentionPeriod"));
-        policy.setPreferredBackupPeriod(jsonObj.getString("PreferredBackupPeriod"));
-        policy.setPreferredBackupTime(jsonObj.getString("PreferredBackupTime"));
-        return policy;
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBinlogFilesRequest.java

@@ -1,55 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql.rds.request;
-
-import java.util.Date;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.util.EntityUtils;
-
-import com.alibaba.fastjson.JSONObject;
-import com.alibaba.fastjson.TypeReference;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.DescribeBinlogFileResult;
-
-/**
- * @author chengjin.lyf on 2018/8/7 下午3:41
- * @since 1.0.25
- */
-public class DescribeBinlogFilesRequest extends AbstractRequest<DescribeBinlogFileResult> {
-
-    public DescribeBinlogFilesRequest(){
-        setVersion("2014-08-15");
-        putQueryString("Action", "DescribeBinlogFiles");
-    }
-
-    public void setRdsInstanceId(String rdsInstanceId) {
-        putQueryString("DBInstanceId", rdsInstanceId);
-    }
-
-    public void setPageSize(int pageSize) {
-        putQueryString("PageSize", String.valueOf(pageSize));
-    }
-
-    public void setPageNumber(int pageNumber) {
-        putQueryString("PageNumber", String.valueOf(pageNumber));
-    }
-
-    public void setStartDate(Date startDate) {
-        putQueryString("StartTime", formatUTCTZ(startDate));
-    }
-
-    public void setEndDate(Date endDate) {
-        putQueryString("EndTime", formatUTCTZ(endDate));
-    }
-
-    public void setResourceOwnerId(Long resourceOwnerId) {
-        putQueryString("ResourceOwnerId", String.valueOf(resourceOwnerId));
-    }
-
-    @Override
-    protected DescribeBinlogFileResult processResult(HttpResponse response) throws Exception {
-        String result = EntityUtils.toString(response.getEntity());
-        DescribeBinlogFileResult describeBinlogFileResult = JSONObject.parseObject(result,
-            new TypeReference<DescribeBinlogFileResult>() {
-            });
-        return describeBinlogFileResult;
-    }
-}

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java

@@ -68,8 +68,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
 
             try {
                 // druid暂时flush privileges语法解析有问题
-                if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")
+                if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")) {
-                    && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "grant")) {
                     repository.console(ddl);
                 }
             } catch (Throwable e) {

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java

@@ -1,119 +0,0 @@
-package com.alibaba.otter.canal.parse.inbound.mysql;
-
-import java.net.InetSocketAddress;
-import java.util.Calendar;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
-import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy;
-import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
-import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
-import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
-import com.alibaba.otter.canal.protocol.CanalEntry;
-import com.alibaba.otter.canal.protocol.position.EntryPosition;
-import com.alibaba.otter.canal.protocol.position.LogPosition;
-import com.alibaba.otter.canal.sink.exception.CanalSinkException;
-
-/**
- * @author chengjin.lyf on 2018/7/21 下午5:24
- * @since 1.0.25
- */
-public class RdsBinlogEventParserProxyTest {
-
-    private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
-    private static final String MYSQL_ADDRESS = "";
-    private static final String USERNAME      = "";
-    private static final String PASSWORD      = "";
-    public static final String  DBNAME        = "";
-    public static final String  TBNAME        = "";
-    public static final String  DDL           = "";
-
-    @Test
-    public void test_timestamp() throws InterruptedException {
-        final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
-        final AtomicLong entryCount = new AtomicLong(0);
-        final EntryPosition entryPosition = new EntryPosition();
-
-        final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
-        Calendar calendar = Calendar.getInstance();
-        calendar.add(Calendar.HOUR_OF_DAY, -24 * 4);
-        final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
-        controller.setSlaveId(3344L);
-        controller.setDetectingEnable(false);
-        controller.setDetectingSQL(DETECTING_SQL);
-        controller.setMasterInfo(buildAuthentication());
-        controller.setMasterPosition(defaultPosition);
-        controller.setInstanceId("");
-        controller.setAccesskey("");
-        controller.setSecretkey("");
-        controller.setDirectory("/tmp/binlog");
-        controller.setEventBlackFilter(new AviaterRegexFilter("mysql\\.*"));
-        controller.setFilterTableError(true);
-        controller.setBatchFileSize(4);
-        controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
-
-            @Override
-            public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
-                                                                                                                   throws CanalSinkException {
-                for (CanalEntry.Entry entry : entrys) {
-                    if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
-                        entryCount.incrementAndGet();
-
-                        String logfilename = entry.getHeader().getLogfileName();
-                        long logfileoffset = entry.getHeader().getLogfileOffset();
-                        long executeTime = entry.getHeader().getExecuteTime();
-
-                        entryPosition.setJournalName(logfilename);
-                        entryPosition.setPosition(logfileoffset);
-                        entryPosition.setTimestamp(executeTime);
-                        break;
-                    }
-                }
-                return true;
-            }
-        });
-
-        controller.setLogPositionManager(new AbstractLogPositionManager() {
-
-            private LogPosition logPosition;
-
-            public void persistLogPosition(String destination, LogPosition logPosition) {
-                this.logPosition = logPosition;
-            }
-
-            public LogPosition getLatestIndexBy(String destination) {
-                return logPosition;
-            }
-        });
-
-        controller.start();
-        timeoutChecker.waitForIdle();
-
-        if (controller.isStart()) {
-            controller.stop();
-        }
-
-        // check
-        Assert.assertTrue(entryCount.get() > 0);
-
-        // 对比第一条数据和起始的position相同
-        Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
-        Assert.assertTrue(entryPosition.getPosition() <= 6163L);
-        Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
-    }
-
-    // ======================== helper method =======================
-
-    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
-        return new EntryPosition(binlogFile, offest, timestamp);
-    }
-
-    private AuthenticationInfo buildAuthentication() {
-        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
-    }
-}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java

@@ -1,14 +1,11 @@
 package com.alibaba.otter.canal.parse.inbound.mysql;
 
+import java.io.File;
 import java.util.Date;
-import java.util.List;
 
 import org.apache.commons.lang.time.DateUtils;
-import org.junit.Test;
 
 import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
-import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
 
 /**
  * @author agapple 2017年10月15日 下午2:14:34
@@ -16,19 +13,15 @@ import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
  */
 public class RdsBinlogOpenApiTest {
 
-    @Test
     public void testSimple() throws Throwable {
-        Date startTime = DateUtils.parseDate("2018-08-10 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" });
+        Date startTime = DateUtils.parseDate("2017-10-13 20:56:58", new String[] { "yyyy-MM-dd HH:mm:ss" });
-        Date endTime = DateUtils.parseDate("2018-08-11 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" });
+        Date endTime = DateUtils.parseDate("2017-10-14 02:57:59", new String[] { "yyyy-MM-dd HH:mm:ss" });
-        String url = "https://rds.aliyuncs.com/";
+        RdsBinlogOpenApi.downloadBinlogFiles("https://rds.aliyuncs.com/",
-        String ak = "";
+            "",
-        String sk = "";
+            "",
-        String dbInstanceId = "";
+            "rm-bp180v4mfjnm157es",
-
+            startTime,
-        RdsBackupPolicy backupPolicy = RdsBinlogOpenApi.queryBinlogBackupPolicy(url, ak, sk, dbInstanceId);
+            endTime,
-        System.out.println(backupPolicy);
+            new File("/tmp/binlog/"));
-
-        List<BinlogFile> binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime);
-        System.out.println(binlogFiles);
     }
 }

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/NoStorageTest.java

@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+
+public class NoStorageTest {
+    final String DBNAME = "testdb";
+    final String TBNAME = "testtb";
+    final String DDL = "CREATE TABLE `testtb` (\n" +
+            "   `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
+            "   `name` varchar(2048) DEFAULT NULL,\n" +
+            "   `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
+            "   `otter_testcol` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
+            "   PRIMARY KEY (`id`)\n" +
+            " ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
+    @Test
+    public void nostorage() {
+        MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
+        TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, null);
+        EntryPosition entryPosition = new EntryPosition();
+        entryPosition.setTimestamp(new Date().getTime());
+        String fullTableName = DBNAME + "." + TBNAME;
+        tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
+        entryPosition.setTimestamp(new Date().getTime() + 1000L);
+        TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
+        assert result.getDdl().equalsIgnoreCase(DDL);
+    }
+}

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tablemeta/StorageTest.java

@@ -0,0 +1,77 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.tablemeta;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaCallback;
+import com.alibaba.otter.canal.parse.inbound.mysql.tablemeta.impl.mysql.MySqlTableMetaStorageFactory;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class StorageTest {
+
+    final String DBNAME = "testdb";
+    final String TBNAME = "testtb";
+    final String DDL = "CREATE TABLE `testtb` (\n" +
+            "   `id` int(11) NOT NULL AUTO_INCREMENT,\n" +
+            "   `name` varchar(2048) DEFAULT NULL,\n" +
+            "   `datachange_lasttime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最晚更新时间',\n" +
+            "   `otter_testcol` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol1` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol2` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol3` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol4` varchar(45) DEFAULT NULL,\n" +
+            "   `otter_testcol5` varchar(45) DEFAULT NULL,\n" +
+            "   PRIMARY KEY (`id`)\n" +
+            " ) ENGINE=InnoDB AUTO_INCREMENT=58333898 DEFAULT CHARSET=utf8mb4";
+
+    @Test
+    public void storage() {
+
+        MySqlTableMetaStorageFactory factory = new MySqlTableMetaStorageFactory(new MySqlTableMetaCallback() {
+            @Override
+            public void save(String dbAddress, String schema, String table, String ddl, Long timestamp) {
+
+            }
+
+            @Override
+            public List<TableMetaEntry> fetch(String dbAddress, String dbName) {
+                TableMetaEntry tableMeta = new TableMetaEntry();
+                tableMeta.setSchema(DBNAME);
+                tableMeta.setTable(TBNAME);
+                tableMeta.setDdl(DDL);
+                tableMeta.setTimestamp(new Date().getTime());
+                List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
+                entries.add(tableMeta);
+                return entries;
+            }
+
+            @Override
+            public List<TableMetaEntry> fetch(String dbAddress, String dbName, String tableName) {
+                TableMetaEntry tableMeta = new TableMetaEntry();
+                tableMeta.setSchema(DBNAME);
+                tableMeta.setTable(TBNAME);
+                tableMeta.setDdl(DDL);
+                tableMeta.setTimestamp(new Date().getTime());
+                List<TableMetaEntry> entries = new ArrayList<TableMetaEntry>();
+                entries.add(tableMeta);
+                return entries;
+            }
+        }, DBNAME);
+        MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello");
+        TableMetaCacheWithStorage tableMetaCacheWithStorage = new TableMetaCacheWithStorage(connection, factory.getTableMetaStorage());
+        EntryPosition entryPosition = new EntryPosition();
+        entryPosition.setTimestamp(new Date().getTime());
+        String fullTableName = DBNAME + "." + TBNAME;
+        tableMetaCacheWithStorage.apply(entryPosition, fullTableName, DDL, null);
+
+        entryPosition.setTimestamp(new Date().getTime() + 1000L);
+        TableMeta result = tableMetaCacheWithStorage.getTableMeta(DBNAME, TBNAME, false, entryPosition);
+        assert result.getDdl().equalsIgnoreCase(DDL);
+    }
+}

pom.xml

@@ -4,7 +4,7 @@
     <artifactId>canal</artifactId>
     <packaging>pom</packaging>
     <name>canal module for otter ${project.version}</name>
-    <version>1.1.0</version>
+    <version>1.0.26-SNAPSHOT</version>
     <url>https://github.com/alibaba/canal</url>
     <parent>
         <groupId>org.sonatype.oss</groupId>

prometheus/pom.xml

@@ -1,15 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.0</version>
+        <version>1.0.26-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.prometheus</artifactId>
-    <version>1.1.0</version>
+    <version>1.0.26-SNAPSHOT</version>
     <name>canal prometheus module for otter ${project.version}</name>
     <dependencies>
         <dependency>

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusService.java

@@ -22,7 +22,6 @@ public class PrometheusService implements CanalMetricsService {
     private static final Logger          logger          = LoggerFactory.getLogger(PrometheusService.class);
     private final CanalInstanceExports   instanceExports;
     private volatile boolean             running         = false;
-    private int                          port;
     private HTTPServer                   server;
     private final ClientInstanceProfiler clientProfiler;
 
@@ -42,9 +41,9 @@ public class PrometheusService implements CanalMetricsService {
     @Override
     public void initialize() {
         try {
-            logger.info("Start prometheus HTTPServer on port {}.", port);
+            //TODO 1.Configurable port
-            //TODO 2.Https?
+            //TODO 2.Https
-            server = new HTTPServer(port);
+            server = new HTTPServer(11112);
         } catch (IOException e) {
             logger.warn("Unable to start prometheus HTTPServer.", e);
             return;
@@ -113,9 +112,4 @@ public class PrometheusService implements CanalMetricsService {
         logger.info("Unregister metrics for destination {}.", instance.getDestination());
     }
 
-    @Override
-    public void setServerPort(int port) {
-        this.port = port;
-    }
-
 }

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java

@@ -7,6 +7,7 @@ import com.alibaba.otter.canal.sink.CanalEventSink;
 import com.alibaba.otter.canal.sink.entry.EntryEventSink;
 import com.google.common.base.Preconditions;
 import io.prometheus.client.Collector;
+import io.prometheus.client.Counter;
 import io.prometheus.client.CounterMetricFamily;
 import io.prometheus.client.GaugeMetricFamily;
 import org.slf4j.Logger;
@@ -29,8 +30,12 @@ public class EntryCollector extends Collector implements InstanceRegistry {
     private static final Logger                             logger            = LoggerFactory.getLogger(SinkCollector.class);
     private static final String                             DELAY             = "canal_instance_traffic_delay";
     private static final String                             TRANSACTION       = "canal_instance_transactions";
+    private static final String                             ROW_EVENTS        = "canal_instance_row_events";
+    private static final String                             ROWS_COUNTER      = "canal_instance_rows_counter";
     private static final String                             DELAY_HELP        = "Traffic delay of canal instance in milliseconds";
     private static final String                             TRANSACTION_HELP  = "Transactions counter of canal instance";
+    private static final String                             ROW_EVENTS_HELP   = "Rowdata events counter of canal instance";
+    private static final String                             ROWS_COUNTER_HELP = "Rows counter of canal instance";
     private final ConcurrentMap<String, EntryMetricsHolder> instances        = new ConcurrentHashMap<String, EntryMetricsHolder>();
 
     private EntryCollector() {}
@@ -50,6 +55,10 @@ public class EntryCollector extends Collector implements InstanceRegistry {
                 DELAY_HELP, DEST_LABELS_LIST);
         CounterMetricFamily transactions = new CounterMetricFamily(TRANSACTION,
                 TRANSACTION_HELP, DEST_LABELS_LIST);
+        CounterMetricFamily rowEvents = new CounterMetricFamily(ROW_EVENTS,
+                ROW_EVENTS_HELP, DEST_LABELS_LIST);
+        CounterMetricFamily rowsCounter = new CounterMetricFamily(ROWS_COUNTER,
+                ROWS_COUNTER_HELP, DEST_LABELS_LIST);
         for (EntryMetricsHolder emh : instances.values()) {
             long now = System.currentTimeMillis();
             long latest = emh.latestExecTime.get();
@@ -57,9 +66,13 @@ public class EntryCollector extends Collector implements InstanceRegistry {
                 delay.addMetric(emh.destLabelValues, (now - latest));
             }
             transactions.addMetric(emh.destLabelValues, emh.transactionCounter.doubleValue());
+            rowEvents.addMetric(emh.destLabelValues, emh.rowEventCounter.doubleValue());
+            rowsCounter.addMetric(emh.destLabelValues, emh.rowsCounter.doubleValue());
         }
         mfs.add(delay);
         mfs.add(transactions);
+        mfs.add(rowEvents);
+        mfs.add(rowsCounter);
         return mfs;
     }
 
@@ -76,8 +89,12 @@ public class EntryCollector extends Collector implements InstanceRegistry {
         PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink);
         holder.latestExecTime = handler.getLatestExecuteTime();
         holder.transactionCounter = handler.getTransactionCounter();
+        holder.rowEventCounter = handler.getRowEventCounter();
+        holder.rowsCounter = handler.getRowsCounter();
         Preconditions.checkNotNull(holder.latestExecTime);
         Preconditions.checkNotNull(holder.transactionCounter);
+        Preconditions.checkNotNull(holder.rowEventCounter);
+        Preconditions.checkNotNull(holder.rowsCounter);
         EntryMetricsHolder old = instances.put(destination, holder);
         if (old != null) {
             logger.warn("Remove stale EntryCollector for instance {}.", destination);
@@ -128,6 +145,8 @@ public class EntryCollector extends Collector implements InstanceRegistry {
     private class EntryMetricsHolder {
         private AtomicLong   latestExecTime;
         private AtomicLong   transactionCounter;
+        private AtomicLong   rowEventCounter;
+        private AtomicLong   rowsCounter;
         private List<String> destLabelValues;
     }
 

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java

@@ -1,10 +1,15 @@
 package com.alibaba.otter.canal.prometheus.impl;
 
-import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
-import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.prometheus.InstanceRegistry;
+import com.google.common.base.Preconditions;
 import io.prometheus.client.Collector;
 import io.prometheus.client.CounterMetricFamily;
 import io.prometheus.client.GaugeMetricFamily;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -14,14 +19,9 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.slf4j.Logger;
+import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST;
-import org.slf4j.LoggerFactory;
+import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST;
-
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import com.alibaba.otter.canal.instance.core.CanalInstance;
-import com.alibaba.otter.canal.parse.CanalEventParser;
-import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
-import com.alibaba.otter.canal.prometheus.InstanceRegistry;
-import com.google.common.base.Preconditions;
 
 /**
  * @author Chuanyi Li

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusCanalEventDownStreamHandler.java

@@ -13,8 +13,10 @@ import java.util.concurrent.atomic.AtomicLong;
  */
 public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDownStreamHandler<List<Event>> {
 
-    private final AtomicLong latestExecuteTime  = new AtomicLong(System.currentTimeMillis());
+    private final AtomicLong latestExecuteTime  = new AtomicLong(0L);
     private final AtomicLong transactionCounter = new AtomicLong(0L);
+    private final AtomicLong rowEventCounter    = new AtomicLong(0L);
+    private final AtomicLong rowsCounter        = new AtomicLong(0L);
 
     @Override
     public List<Event> before(List<Event> events) {
@@ -32,6 +34,8 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow
                     case ROWDATA: {
                         long exec = e.getExecuteTime();
                         if (exec > 0) localExecTime = exec;
+                        rowEventCounter.incrementAndGet();
+                        rowsCounter.addAndGet(e.getRowsCount());
                         break;
                     }
                     case TRANSACTIONEND: {
@@ -76,4 +80,11 @@ public class PrometheusCanalEventDownStreamHandler extends AbstractCanalEventDow
         return transactionCounter;
     }
 
+    public AtomicLong getRowsCounter() {
+        return rowsCounter;
+    }
+
+    public AtomicLong getRowEventCounter() {
+        return rowEventCounter;
+    }
 }

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/PrometheusClientInstanceProfiler.java

@@ -71,6 +71,7 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler
         String destination = result.getDestination();
         PacketType type = result.getType();
         outboundCounter.labels(destination).inc(result.getAmount());
+        packetsCounter.labels(destination, type.name()).inc();
         short errorCode = result.getErrorCode();
         if (errorCode > 0) {
             errorsCounter.labels(destination, Short.toString(errorCode)).inc();
@@ -80,16 +81,12 @@ public class PrometheusClientInstanceProfiler implements ClientInstanceProfiler
         switch (type) {
             case GET:
                 boolean empty = result.getEmpty();
-                // 区分一下空包
                 if (empty) {
                     emptyBatchesCounter.labels(destination).inc();
-                } else {
-                    packetsCounter.labels(destination, type.name()).inc();
                 }
                 break;
             // reserve for others
             default:
-                packetsCounter.labels(destination, type.name()).inc();
                 break;
         }
     }

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java

@@ -34,25 +34,13 @@ public class StoreCollector extends Collector implements InstanceRegistry {
     private static final String                             STORE            = "canal_instance_store";
     private static final String                             PRODUCE_MEM      = "canal_instance_store_produce_mem";
     private static final String                             CONSUME_MEM      = "canal_instance_store_consume_mem";
-    private static final String                             PUT_DELAY        = "canal_instance_put_delay";
-    private static final String                             GET_DELAY        = "canal_instance_get_delay";
-    private static final String                             ACK_DELAY        = "canal_instance_ack_delay";
-    private static final String                             PUT_ROWS         = "canal_instance_put_rows";
-    private static final String                             GET_ROWS         = "canal_instance_get_rows";
-    private static final String                             ACK_ROWS         = "canal_instance_ack_rows";
     private static final String                             PRODUCE_HELP     = "Produced events counter of canal instance";
     private static final String                             CONSUME_HELP     = "Consumed events counter of canal instance";
     private static final String                             STORE_HELP       = "Canal instance info";
     private static final String                             PRODUCE_MEM_HELP = "Produced mem bytes of canal instance";
     private static final String                             CONSUME_MEM_HELP = "Consumed mem bytes of canal instance";
-    private static final String                             PUT_DELAY_HELP   = "Traffic delay of canal instance put";
-    private static final String                             GET_DELAY_HELP   = "Traffic delay of canal instance get";
-    private static final String                             ACK_DELAY_HELP   = "Traffic delay of canal instance ack";
-    private static final String                             PUT_ROWS_HELP    = "Put table rows of canal instance";
-    private static final String                             GET_ROWS_HELP    = "Got table rows of canal instance";
-    private static final String                             ACK_ROWS_HELP    = "Acked table rows of canal instance";
     private final ConcurrentMap<String, StoreMetricsHolder> instances        = new ConcurrentHashMap<String, StoreMetricsHolder>();
-    private final List<String>                              storeLabelsList  = Arrays.asList(DEST, "batchMode", "size");
+    private final List<String>                              storeLabelsList  = Arrays.asList(DEST, "batchMode");
 
     private StoreCollector() {}
 
@@ -77,40 +65,11 @@ public class StoreCollector extends Collector implements InstanceRegistry {
                 PRODUCE_MEM_HELP, DEST_LABELS_LIST);
         CounterMetricFamily ackMem = new CounterMetricFamily(CONSUME_MEM,
                 CONSUME_MEM_HELP, DEST_LABELS_LIST);
-        GaugeMetricFamily putDelay = new GaugeMetricFamily(PUT_DELAY,
-                PUT_DELAY_HELP, DEST_LABELS_LIST);
-        GaugeMetricFamily getDelay = new GaugeMetricFamily(GET_DELAY,
-                GET_DELAY_HELP, DEST_LABELS_LIST);
-        GaugeMetricFamily ackDelay = new GaugeMetricFamily(ACK_DELAY,
-                ACK_DELAY_HELP, DEST_LABELS_LIST);
-        CounterMetricFamily putRows = new CounterMetricFamily(PUT_ROWS,
-                PUT_ROWS_HELP, DEST_LABELS_LIST);
-        CounterMetricFamily getRows = new CounterMetricFamily(GET_ROWS,
-                GET_ROWS_HELP, DEST_LABELS_LIST);
-        CounterMetricFamily ackRows = new CounterMetricFamily(ACK_ROWS,
-                ACK_ROWS_HELP, DEST_LABELS_LIST);
         boolean hasMem = false;
         for (StoreMetricsHolder smh : instances.values()) {
             final boolean isMem = smh.batchMode.isMemSize();
             put.addMetric(smh.destLabelValues, smh.putSeq.doubleValue());
             ack.addMetric(smh.destLabelValues, smh.ackSeq.doubleValue());
-            long pet = smh.putExecTime.get();
-            // 防止出现启动时，未消费造成的get, ack延时小于前阶段的情况
-            long get = Math.min(smh.getExecTime.get(), pet);
-            long aet = Math.min(smh.ackExecTime.get(), get);
-            long now = System.currentTimeMillis();
-            if (now >= pet) {
-                putDelay.addMetric(smh.destLabelValues, (now - pet));
-            }
-            if (now >= get) {
-                getDelay.addMetric(smh.destLabelValues, (now - get));
-            }
-            if (now >= aet) {
-                ackDelay.addMetric(smh.destLabelValues, (now - aet));
-            }
-            putRows.addMetric(smh.destLabelValues, smh.putTableRows.doubleValue());
-            getRows.addMetric(smh.destLabelValues, smh.getTableRows.doubleValue());
-            ackRows.addMetric(smh.destLabelValues, smh.ackTableRows.doubleValue());
             store.addMetric(smh.storeLabelValues, 1);
             if (isMem) {
                 hasMem = true;
@@ -121,12 +80,6 @@ public class StoreCollector extends Collector implements InstanceRegistry {
         mfs.add(put);
         mfs.add(ack);
         mfs.add(store);
-        mfs.add(putDelay);
-        mfs.add(getDelay);
-        mfs.add(ackDelay);
-        mfs.add(putRows);
-        mfs.add(getRows);
-        mfs.add(ackRows);
         if (hasMem) {
             mfs.add(putMem);
             mfs.add(ackMem);
@@ -147,14 +100,7 @@ public class StoreCollector extends Collector implements InstanceRegistry {
         holder.putSeq = memStore.getPutSequence();
         holder.ackSeq = memStore.getAckSequence();
         holder.destLabelValues = Collections.singletonList(destination);
-        holder.size = memStore.getBufferSize();
+        holder.storeLabelValues = Arrays.asList(destination, memStore.getBatchMode().name());
-        holder.storeLabelValues = Arrays.asList(destination, holder.batchMode.name(), Integer.toString(holder.size));
-        holder.putExecTime = memStore.getPutExecTime();
-        holder.getExecTime = memStore.getGetExecTime();
-        holder.ackExecTime = memStore.getAckExecTime();
-        holder.putTableRows = memStore.getPutTableRows();
-        holder.getTableRows = memStore.getGetTableRows();
-        holder.ackTableRows = memStore.getAckTableRows();
         Preconditions.checkNotNull(holder.batchMode);
         Preconditions.checkNotNull(holder.putSeq);
         Preconditions.checkNotNull(holder.ackSeq);
@@ -182,13 +128,6 @@ public class StoreCollector extends Collector implements InstanceRegistry {
         private BatchMode    batchMode;
         private AtomicLong   putMemSize;
         private AtomicLong   ackMemSize;
-        private AtomicLong   putExecTime;
-        private AtomicLong   getExecTime;
-        private AtomicLong   ackExecTime;
-        private AtomicLong   putTableRows;
-        private AtomicLong   getTableRows;
-        private AtomicLong   ackTableRows;
-        private int          size;
         private List<String> destLabelValues;
         private List<String> storeLabelValues;
     }

protocol/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

server/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.0</version>
+		<version>1.0.26-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<artifactId>canal.server</artifactId>

server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java

@@ -44,7 +44,6 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
     private Map<String, CanalInstance> canalInstances;
     // private Map<ClientIdentity, Position> lastRollbackPostions;
     private CanalInstanceGenerator     canalInstanceGenerator;
-    private int                        metricsPort;
     private CanalMetricsService        metrics          = NopCanalMetricsService.NOP;
 
     private static class SingletonHolder {
@@ -65,7 +64,6 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
             super.start();
             // 如果存在provider,则启动metrics service
             loadCanalMetrics();
-            metrics.setServerPort(metricsPort);
             metrics.initialize();
             canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>() {
 
@@ -531,8 +529,4 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
         this.canalInstanceGenerator = canalInstanceGenerator;
     }
 
-    public void setMetricsPort(int metricsPort) {
-        this.metricsPort = metricsPort;
-    }
-
 }

server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java

@@ -220,7 +220,6 @@ public class SessionHandler extends SimpleChannelHandler {
                         } else {
                             clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
                             embeddedServer.ack(clientIdentity, ack.getBatchId());
-                            new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
                         }
                     } else {
                         byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
@@ -240,7 +239,6 @@ public class SessionHandler extends SimpleChannelHandler {
                         } else {
                             embeddedServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次
                         }
-                        new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
                     } else {
                         byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage());
                         NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(rollback.getDestination(),

server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java

@@ -7,7 +7,6 @@ import org.jboss.netty.channel.ChannelFuture;
 import org.jboss.netty.channel.ChannelFutureListener;
 
 import static com.alibaba.otter.canal.server.netty.CanalServerWithNettyProfiler.profiler;
-import static com.alibaba.otter.canal.server.netty.NettyUtils.HEADER_LENGTH;
 
 /**
  * @author Chuanyi Li
@@ -33,7 +32,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
                 .destination(destination)
                 .type(type)
                 .request(request)
-                .amount(amount + HEADER_LENGTH)
+                .amount(amount)
                 .latency(latency)
                 .errorCode(errorCode)
                 .empty(empty)
@@ -43,7 +42,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
     @Override
     public void operationComplete(ChannelFuture future) {
         // profiling after I/O operation
-        if (future != null && future.getCause() != null) {
+        if (future.getCause() != null) {
             result.channelError = future.getCause();
         }
         profiler().profiling(result);