chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge tsdb meta support and refactor

Browse Source
This commit is contained in:
agapple
2017-09-30 23:21:17 +08:00
parent 3930ea0f9b
commit 19707f92d4
45 changed files with 585 additions and 899 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java
View File

@@ -2,34 +2,43 @@ package com.alibaba.otter.canal.parse.inbound;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.List;
import com.taobao.tddl.dbsync.binlog.BinlogPosition;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.druid.sql.visitor.functions.Bin;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
public class TableMetaCacheTest {
//@Test
//public void testSimple() {
//
// MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
// try {
// connection.connect();
// } catch (IOException e) {
// Assert.fail(e.getMessage());
// }
//
// TableMetaCache cache = new TableMetaCache(connection);
// TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
// Assert.assertNotNull(meta);
// for (FieldMeta field : meta.getFields()) {
// System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : "
// + field.isNullable());
// }
//}
@Test
public void testSimple() throws IOException {
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("100.81.153.8", 3308),
"jingwei",
"jingwei");
try {
connection.connect();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
List<ResultSetPacket> packets = connection.queryMulti("show create table test.ljh_test; show create table test.user");
String createDDL = null;
if (packets.get(0).getFieldValues().size() > 0) {
createDDL = packets.get(0).getFieldValues().get(1);
}
System.out.println(createDDL);
// TableMetaCache cache = new TableMetaCache(connection);
// TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
// Assert.assertNotNull(meta);
// for (FieldMeta field : meta.getFields()) {
// System.out.println("filed :" + field.getColumnName() + " , isKey : "
// + field.isKey() + " , isNull : "
// + field.isNullable());
// }
}
}

1
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java
View File

@@ -88,7 +88,6 @@ public class GroupEventPaserTest {
private BinlogParser buildParser(AuthenticationInfo info) {
return new AbstractBinlogParser<LogEvent>() {
@Override
public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException {
return null;

4
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java
View File

@@ -3,8 +3,8 @@ package com.alibaba.otter.canal.parse.inbound.mysql;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult;
import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult;
import com.alibaba.otter.canal.parse.inbound.mysql.ddl.SimpleDdlParser;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
public class SimpleDdlParserTest {

4
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java
View File

@@ -19,7 +19,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
* @author agapple 2017年8月1日 下午7:15:54
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"/dal-dao.xml"})
@ContextConfiguration(locations = { "/dal-dao.xml" })
public class MemoryTableMetaTest {
@Test
@@ -33,6 +33,6 @@ public class MemoryTableMetaTest {
TableMeta meta = memoryTableMeta.find("test", "test");
System.out.println(meta);
Assert.assertTrue( meta.getFieldMetaByName("ID").isKey());
Assert.assertTrue(meta.getFieldMetaByName("ID").isKey());
}
}

9
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java
View File

@@ -13,12 +13,10 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* Created by wanshao
* Date: 2017/9/20
* Time: 下午5:00
* Created by wanshao Date: 2017/9/20 Time: 下午5:00
**/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"/dal-dao.xml"})
@ContextConfiguration(locations = { "/dal-dao.xml" })
public class MetaHistoryDAOTest {
@Resource
@@ -26,8 +24,7 @@ public class MetaHistoryDAOTest {
@Test
public void testGetAll() {
List<MetaHistoryDO>
metaHistoryDOList = metaHistoryDAO.getAll();
List<MetaHistoryDO> metaHistoryDOList = metaHistoryDAO.getAll();
for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
System.out.println(metaHistoryDO.getId());
}

2
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java
View File

@@ -30,8 +30,6 @@ public class TableMetaManagerTest {
@Test
public void testSimple() throws FileNotFoundException, IOException {
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
File create = new File(dummyFile.getParent() + "/ddl", "create.sql");

19
parse/src/test/resources/dal-dao.xml → parse/src/test/resources/tsdb/dal-dao.xml
View File

@@ -9,13 +9,10 @@
<bean id="dataSource" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DataSourceFactoryTSDB"
factory-method="getDataSource">
<constructor-arg index="0" value="127.0.0.1"/>
<constructor-arg index="1" value="canal"/>
<constructor-arg index="2" value="canal"/>
<constructor-arg index="3" value="true"/>
<constructor-arg index="4" value="tsdb"/>
<constructor-arg index="5" value="jdbc:mysql://127.0.0.1:3306/tsdb"/>
<constructor-arg index="6" value="com.mysql.jdbc.Driver"/>
<constructor-arg index="0" value="${canal.instance.tsdb.url}"/>
<constructor-arg index="1" value="${canal.instance.tsdb.dbUsername}"/>
<constructor-arg index="2" value="${canal.instance.tsdb.dbPassword}"/>
<constructor-arg index="3" value="${canal.instance.tsdb.enable:true}"/>
</bean>
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
@@ -32,15 +29,11 @@
<property name="configLocation" value="classpath:sqlmap-config.xml"/>
</bean>
<bean id="MetaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<bean id="metaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="MetaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<bean id="metaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="PersistTableMeta" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager"></bean>
</beans>

20
parse/src/test/resources/tsdb/mybatis-config.xml Normal file
View File

@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<environments default="development">
<environment id="development">
<transactionManager type="JDBC"/>
<dataSource type="POOLED">
<property name="driver" value="com.mysql.jdbc.Driver"/>
<property name="url" value="jdbc:mysql://127.0.0.1:3306/tsdb"/>
<property name="username" value="canal"/>
<property name="password" value="canal"/>
</dataSource>
</environment>
</environments>
<mappers>
<package name="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.mapper"/>
</mappers>
</configuration>

57
parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml Normal file
View File

@@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
<sqlMap namespace="table_meta_history">
<typeAlias alias="metaHistoryDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO"/>
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,use_schema,`schema`,`table`,`sql`,`type`,`extra`
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMasterId,a.binlog_timestamp as binlogTimestamp,
a.use_schema as useSchema,a.`schema` as `schema`,a.`table` as `table`,a.`sql` as `sql`,a.`type` as `type`,a.`extra` as `extra`
]]>
</sql>
<select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaHistoryDO">
select
<include refid="allVOColumns"/>
from `canal_table_meta_history$env$` a
<![CDATA[
where binlog_timestamp >= #snapshotTimestamp# and binlog_timestamp <= #timestamp#
order by binlog_timestamp asc,id asc
]]>
</select>
<insert id="insert" parameterClass="metaHistoryDO">
insert into `canal_table_meta_history` (<include refid="allColumns"/>)
values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#schema#,#table#,#sql#,#type#,#extra#);
<selectKey resultClass="java.lang.Long" keyProperty="id">
SELECT last_insert_id()
</selectKey>
</insert>
<delete id="deleteByGmtModified" parameterClass="java.util.Map">
<![CDATA[
delete from `canal_table_meta_history`
where gmt_modified < date_sub(now(),interval #interval# second)
]]>
</delete>
<select id="getAll" resultClass="metaHistoryDO">
select * from canal_table_meta_history
</select>
</sqlMap>

66
parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml Normal file
View File

@@ -0,0 +1,66 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE sqlMap PUBLIC "-//ibatis.apache.org//DTD SQL Map 2.0//EN" "http://ibatis.apache.org/dtd/sql-map-2.dtd" >
<sqlMap namespace="table_meta_snapshot">
<typeAlias alias="metaSnapshotDO" type="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO"/>
<typeAlias alias="tableMetaSnapshotDO"
type="com.alibaba.middleware.jingwei.biz.dataobject.CanalTableMetaSnapshotDO"/>
<sql id="allColumns">
<![CDATA[
gmt_create,gmt_modified,binlog_file,binlog_offest,binlog_master_id,binlog_timestamp,data,extra
]]>
</sql>
<sql id="allVOColumns">
<![CDATA[
a.id as id,a.gmt_create as gmtCreate,a.gmt_modified as gmtModified,
a.binlog_file as binlogFile,a.binlog_offest as binlogOffest,a.binlog_master_id as binlogMaster_id,a.binlog_timestamp as binlogTimestamp,a.data as data,a.extra as extra
]]>
</sql>
<select id="findByTimestamp" parameterClass="java.util.Map" resultClass="metaSnapshotDO">
select
<include refid="allVOColumns"/>
from `canal_table_meta_snapshot$env$` a
<![CDATA[
where binlog_timestamp < #timestamp#
order by binlog_timestamp desc,id desc
limit 1
]]>
</select>
<insert id="insert" parameterClass="metaSnapshotDO">
insert into `canal_table_meta_snapshot` (<include refid="allColumns"/>)
values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#);
<selectKey resultClass="java.lang.Long" keyProperty="id">
SELECT last_insert_id()
</selectKey>
</insert>
<update id="update" parameterClass="metaSnapshotDO">
update `canal_table_meta_snapshot` set gmt_modified=now(),
binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra#
where binlog_timestamp=0
</update>
<delete id="deleteByGmtModified" parameterClass="java.util.Map">
<![CDATA[
delete from `canal_table_meta_snapshot`
where gmt_modified < date_sub(now(),interval #interval# second)
]]>
</delete>
<select id="getAll" resultClass="metaSnapshotDO">
select * from canal_table_meta_snapshot
</select>
</sqlMap>

9
parse/src/test/resources/tsdb/sqlmap-config.xml Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE sqlMapConfig PUBLIC "-//iBATIS.com//DTD SQL Map Config 2.0//EN"
"http://www.ibatis.com/dtd/sql-map-config-2.dtd">
<sqlMapConfig>
<settings useStatementNamespaces="true"/>
<sqlMap resource="sql-map/sqlmap_history.xml"/>
<sqlMap resource="sql-map/sqlmap_snapshot.xml"/>
</sqlMapConfig>