chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed tsdb test

Browse Source
This commit is contained in:
agapple
2017-10-14 00:12:43 +08:00
parent 19707f92d4
commit 4f00f5ea8d
55 changed files with 1002 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parse/src/test/resources/ddl/create.sql
View File

@@ -1,5 +1,5 @@
CREATE TABLE `test` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
`id` bigint(20) zerofill unsigNed NOT NULL AUTO_INCREMENT COMMENT 'id',
`c_tinyint` tinyint(4) DEFAULT '1' COMMENT 'tinyint',
`c_smallint` smallint(6) DEFAULT 0 COMMENT 'smallint',
`c_mediumint` mediumint(9) DEFAULT NULL COMMENT 'mediumint',

39
parse/src/test/resources/tsdb/dal-dao.xml
View File

@@ -1,39 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.springframework.org/schema/beans" xmlns:tx="http://www.springframework.org/schema/tx"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx-2.0.xsd"
default-autowire="byName">
<tx:annotation-driven/>
<bean id="dataSource" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DataSourceFactoryTSDB"
factory-method="getDataSource">
<constructor-arg index="0" value="${canal.instance.tsdb.url}"/>
<constructor-arg index="1" value="${canal.instance.tsdb.dbUsername}"/>
<constructor-arg index="2" value="${canal.instance.tsdb.dbPassword}"/>
<constructor-arg index="3" value="${canal.instance.tsdb.enable:true}"/>
</bean>
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="dataSource"/>
</bean>
<bean id="txTemplate" class="org.springframework.transaction.support.TransactionTemplate">
<property name="transactionManager" ref="transactionManager"></property>
<property name="propagationBehaviorName" value="PROPAGATION_REQUIRED"></property>
</bean>
<bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:sqlmap-config.xml"/>
</bean>
<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
</beans>

44
parse/src/test/resources/tsdb/derby-tsdb.xml Normal file
View File

@@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8" ?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.springframework.org/schema/beans" xmlns:tx="http://www.springframework.org/schema/tx"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx-2.0.xsd"
default-autowire="byName">
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
<property name="driverClassName" value="org.apache.derby.jdbc.EmbeddedDriver" />
<property name="url" value="jdbc:derby:derbyTest;create=true" />
<property name="username" value="canal" />
<property name="password" value="canal" />
<property name="maxActive" value="30" />
<property name="initialSize" value="0" />
<property name="minIdle" value="1" />
<property name="maxWait" value="10000" />
<property name="timeBetweenEvictionRunsMillis" value="60000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="useUnfairLock" value="true" />
</bean>
<bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:tsdb/sql-map/sqlmap-config.xml"/>
</bean>
<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
</beans>

20
parse/src/test/resources/tsdb/mybatis-config.xml
View File

@@ -1,20 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<environments default="development">
<environment id="development">
<transactionManager type="JDBC"/>
<dataSource type="POOLED">
<property name="driver" value="com.mysql.jdbc.Driver"/>
<property name="url" value="jdbc:mysql://127.0.0.1:3306/tsdb"/>
<property name="username" value="canal"/>
<property name="password" value="canal"/>
</dataSource>
</environment>
</environments>
<mappers>
<package name="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.mapper"/>
</mappers>
</configuration>

47
parse/src/test/resources/tsdb/mysql-tsdb.xml Normal file
View File

@@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8" ?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.springframework.org/schema/beans" xmlns:tx="http://www.springframework.org/schema/tx"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx-2.0.xsd"
default-autowire="byName">
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>
<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://127.0.0.1:3306/canal_tsdb" />
<property name="username" value="canal" />
<property name="password" value="canal" />
<property name="maxActive" value="30" />
<property name="initialSize" value="0" />
<property name="minIdle" value="1" />
<property name="maxWait" value="10000" />
<property name="timeBetweenEvictionRunsMillis" value="60000" />
<property name="minEvictableIdleTimeMillis" value="300000" />
<property name="validationQuery" value="SELECT 1" />
<property name="exceptionSorterClassName" value="com.alibaba.druid.pool.vendor.MySqlExceptionSorter" />
<property name="validConnectionCheckerClassName" value="com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker" />
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="useUnfairLock" value="true" />
</bean>
<bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:tsdb/sql-map/sqlmap-config.xml"/>
</bean>
<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
</beans>

5
parse/src/test/resources/tsdb/sqlmap-config.xml → parse/src/test/resources/tsdb/sql-map/sqlmap-config.xml
View File

@@ -3,7 +3,6 @@
"http://www.ibatis.com/dtd/sql-map-config-2.dtd">
<sqlMapConfig>
<settings useStatementNamespaces="true"/>
<sqlMap resource="sql-map/sqlmap_history.xml"/>
<sqlMap resource="sql-map/sqlmap_snapshot.xml"/>
<sqlMap resource="tsdb/sql-map/sqlmap_history.xml"/>
<sqlMap resource="tsdb/sql-map/sqlmap_snapshot.xml"/>
</sqlMapConfig>

44
parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml
View File

@@ -1,57 +1,45 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
<sqlMap namespace="table_meta_history">
<typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO"/>
<sqlMap namespace="meta_history">
<typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO"/>
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,`schema`,`table`,`sql`,`type`,`extra`
gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,sql_schema,sql_table,sql_text,sql_type,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
a.use_schema as useSchema,a.`schema` as `schema`,a.`table` as `table`,a.`sql` as `sql`,a.`type` as `type`,a.`extra` as `extra`
a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
a.use_schema as useSchema,a.sql_schema as sqlSchema,a.sql_table as sqlTable,a.sql_text as sqlText,a.sql_type as sqlType,a.extra as extra
]]>
</sql>
<select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaHistoryDO">
select
<include refid="allVOColumns"/>
from `canal_table_meta_history$env$` a
from meta_history a
<![CDATA[
where binlog_timestamp >= #snapshotTimestamp# and binlog_timestamp <= #timestamp#
where destination = #destination# and binlog_timestamp >= #snapshotTimestamp# and binlog_timestamp <= #timestamp#
order by binlog_timestamp asc,id asc
]]>
</select>
<insert id="insert" parameterClass="metaHistoryDO">
insert into `canal_table_meta_history` (<include refid="allColumns"/>)
values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#schema#,#table#,#sql#,#type#,#extra#);
<selectKey resultClass="java.lang.Long" keyProperty="id">
SELECT last_insert_id()
</selectKey>
insert into meta_history (<include refid="allColumns"/>)
values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#sqlSchema#,#sqlTable#,#sqlText#,#sqlType#,#extra#)
</insert>
<delete id="deleteByName" parameterClass="java.util.Map">
delete from meta_history
where destination=#destination#
</delete>
<delete id="deleteByGmtModified" parameterClass="java.util.Map">
<![CDATA[
delete from `canal_table_meta_history`
where gmt_modified < date_sub(now(),interval #interval# second)
delete from meta_history
where gmt_modified < timestamp(#timestamp#)
]]>
</delete>
<select id="getAll" resultClass="metaHistoryDO">
select * from canal_table_meta_history
</select>
</sqlMap>

63
parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml
View File

@@ -1,66 +1,63 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
<sqlMap namespace="table_meta_snapshot">
<typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO"/>
<sqlMap namespace="meta_snapshot">
<typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO"/>
<typeAlias alias="tableMetaSnapshotDO"
type="com.alibaba.middleware.jingwei.biz.dataobject.CanalTableMetaSnapshotDO"/>
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
gmt_create,gmt_modified,destination,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMaster_id,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
a.destination as destination,a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
]]>
</sql>
<select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaSnapshotDO">
select
<include refid="allVOColumns"/>
from `canal_table_meta_snapshot$env$` a
<![CDATA[
where binlog_timestamp < #timestamp#
select <include refid="allVOColumns"/>
<![CDATA[
from meta_snapshot a
where destination = #destination# and binlog_timestamp < #timestamp#
order by binlog_timestamp desc,id desc
limit 1
]]>
</select>
<select id="findByTimestampOnDerby" parameterClass="java.util.Map" resultClass="metaSnapshotDO">
select * FROM (
select ROW_NUMBER() OVER() AS rownum, <include refid="allVOColumns"/>
<![CDATA[
from meta_snapshot a
where destination = #destination# and binlog_timestamp < #timestamp#
order by binlog_timestamp desc,id desc
) AS tmp
WHERE rownum <= 5
]]>
</select>
<insert id="insert" parameterClass="metaSnapshotDO">
insert into `canal_table_meta_snapshot` (<include refid="allColumns"/>)
values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#);
<selectKey resultClass="java.lang.Long" keyProperty="id">
SELECT last_insert_id()
</selectKey>
insert into meta_snapshot (<include refid="allColumns"/>)
values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#)
</insert>
<update id="update" parameterClass="metaSnapshotDO">
update `canal_table_meta_snapshot` set gmt_modified=now(),
update meta_snapshot set gmt_modified=now(),
binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra#
where binlog_timestamp=0
where destination=#destination# and binlog_timestamp=0
</update>
<delete id="deleteByName" parameterClass="java.util.Map">
delete from meta_snapshot
where destination=#destination#
</delete>
<delete id="deleteByGmtModified" parameterClass="java.util.Map">
<![CDATA[
delete from `canal_table_meta_snapshot`
where gmt_modified < date_sub(now(),interval #interval# second)
delete from meta_snapshot
where gmt_modified < timestamp(#timestamp#)
]]>
</delete>
<select id="getAll" resultClass="metaSnapshotDO">
select * from canal_table_meta_snapshot
</select>
</sqlMap>