chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

table meta support

Browse Source
This commit is contained in:
kaimingwan
2017-09-22 22:01:57 +08:00
committed by wanshao
parent 2cebfa04dd
commit e8da1c884b
55 changed files with 2931 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java
View File

@@ -3,31 +3,33 @@ package com.alibaba.otter.canal.parse.inbound;
import java.io.IOException;
import java.net.InetSocketAddress;
import com.taobao.tddl.dbsync.binlog.BinlogPosition;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.druid.sql.visitor.functions.Bin;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
public class TableMetaCacheTest {
@Test
public void testSimple() {
MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
try {
connection.connect();
} catch (IOException e) {
Assert.fail(e.getMessage());
}
TableMetaCache cache = new TableMetaCache(connection);
TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
Assert.assertNotNull(meta);
for (FieldMeta field : meta.getFileds()) {
System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : "
+ field.isNullable());
}
}
//@Test
//public void testSimple() {
//
// MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
// try {
// connection.connect();
// } catch (IOException e) {
// Assert.fail(e.getMessage());
// }
//
// TableMetaCache cache = new TableMetaCache(connection);
// TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
// Assert.assertNotNull(meta);
// for (FieldMeta field : meta.getFields()) {
// System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : "
// + field.isNullable());
// }
//}
}

5
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java
View File

@@ -88,8 +88,9 @@ public class GroupEventPaserTest {
private BinlogParser buildParser(AuthenticationInfo info) {
return new AbstractBinlogParser<LogEvent>() {
public Entry parse(LogEvent event) throws CanalParseException {
// return _parser.parse(event);
@Override
public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException {
return null;
}
};

6
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java
View File

@@ -25,11 +25,11 @@ public class LocalBinlogDumpTest {
@Test
public void testSimple() {
String directory = "/home/jianghang/tmp/binlog";
String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.000006", 4L);
final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setDirectory(directory);
controller.setMasterPosition(startPosition);

16
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java
View File

@@ -23,15 +23,15 @@ import com.alibaba.otter.canal.sink.exception.CanalSinkException;
public class LocalBinlogEventParserTest {
private static final String MYSQL_ADDRESS = "127.0.0.1";
private static final String USERNAME = "xxxxx";
private static final String PASSWORD = "xxxxx";
private static final String USERNAME = "canal";
private static final String PASSWORD = "canal";
private String directory;
@Before
public void setUp() {
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
directory = new File(dummyFile.getParent() + "/binlog").getPath();
directory = new File("/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb").getPath();
}
@Test
@@ -40,7 +40,7 @@ public class LocalBinlogEventParserTest {
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", 219L, 1505467103000L);
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
@@ -103,7 +103,7 @@ public class LocalBinlogEventParserTest {
final AtomicLong entryCount = new AtomicLong(0);
final EntryPosition entryPosition = new EntryPosition();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, 1322803601000L);
final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", null, 1505467103000L);
final LocalBinlogEventParser controller = new LocalBinlogEventParser();
controller.setMasterPosition(defaultPosition);
controller.setMasterInfo(buildAuthentication());
@@ -156,15 +156,15 @@ public class LocalBinlogEventParserTest {
Assert.assertTrue(entryCount.get() > 0);
// 对比第一条数据和起始的position相同
Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
Assert.assertTrue(entryPosition.getPosition() <= 6163L);
Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000003");
Assert.assertTrue(entryPosition.getPosition() <= 300L);
Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
@Test
public void test_no_position() throws InterruptedException {
final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000);
final EntryPosition defaultPosition = buildPosition("mysql-bin.000002",
final EntryPosition defaultPosition = buildPosition("mysql-bin.000003",
null,
new Date().getTime() + 1000 * 1000L);
final AtomicLong entryCount = new AtomicLong(0);

4
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
View File

@@ -26,12 +26,12 @@ public class MysqlDumpTest {
@Test
public void testSimple() {
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L);
final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
controller.setMasterPosition(startPosition);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

6
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java
View File

@@ -24,8 +24,8 @@ public class MysqlEventParserTest {
private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
private static final String MYSQL_ADDRESS = "127.0.0.1";
private static final String USERNAME = "root";
private static final String PASSWORD = "xxxxxx";
private static final String USERNAME = "canal";
private static final String PASSWORD = "canal";
@Test
public void test_position() throws InterruptedException {
@@ -34,7 +34,7 @@ public class MysqlEventParserTest {
final EntryPosition entryPosition = new EntryPosition();
final MysqlEventParser controller = new MysqlEventParser();
final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", 4690L, 1505481064000L);
controller.setSlaveId(3344L);
controller.setDetectingEnable(true);

38
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java Normal file
View File

@@ -0,0 +1,38 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.io.File;
import java.io.FileInputStream;
import java.net.URL;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author agapple 2017年8月1日 下午7:15:54
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"/dal-dao.xml"})
public class MemoryTableMetaTest {
@Test
public void testSimple() throws Throwable {
MemoryTableMeta memoryTableMeta = new MemoryTableMeta(null);
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
memoryTableMeta.apply(null, "test", sql);
TableMeta meta = memoryTableMeta.find("test", "test");
System.out.println(meta);
Assert.assertTrue( meta.getFieldMetaByName("ID").isKey());
}
}

36
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java Normal file
View File

@@ -0,0 +1,36 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.util.List;
import javax.annotation.Resource;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* Created by wanshao
* Date: 2017/9/20
* Time: 下午5:00
**/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"/dal-dao.xml"})
public class MetaHistoryDAOTest {
@Resource
MetaHistoryDAO metaHistoryDAO;
@Test
public void testGetAll() {
List<MetaHistoryDO>
metaHistoryDOList = metaHistoryDAO.getAll();
for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
System.out.println(metaHistoryDO.getId());
}
}
}

47
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java Normal file
View File

@@ -0,0 +1,47 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import javax.annotation.Resource;
import com.taobao.tddl.dbsync.binlog.BinlogPosition;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author wanshao 2017年8月2日 下午4:11:45
* @since 3.2.5
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = { "/dal-dao.xml" })
public class TableMetaManagerTest {
@Resource
TableMetaManager tableMetaManager;
@Test
public void testSimple() throws FileNotFoundException, IOException {
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
BinlogPosition position = BinlogPosition.parseFromString("001115:0139177334#3065927853.1501660815000");
String createSql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
tableMetaManager.apply(position, "tddl5_00", createSql);
String alterSql = "alter table `test` add column name varchar(32) after c_varchar";
position = BinlogPosition.parseFromString("001115:0139177334#3065927853.1501660816000");
tableMetaManager.apply(position, "tddl5_00", alterSql);
}
}

BIN
parse/src/test/resources/binlog/tsdb/mysql-bin.000001 Normal file
View File

Binary file not shown.

BIN
parse/src/test/resources/binlog/tsdb/mysql-bin.000002 Normal file
View File

Binary file not shown.

BIN
parse/src/test/resources/binlog/tsdb/mysql-bin.000003 Normal file
View File

Binary file not shown.

46
parse/src/test/resources/dal-dao.xml Normal file
View File

@@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8" ?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.springframework.org/schema/beans" xmlns:tx="http://www.springframework.org/schema/tx"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx-2.0.xsd"
default-autowire="byName">
<tx:annotation-driven/>
<bean id="dataSource" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DataSourceFactoryTSDB"
factory-method="getDataSource">
<constructor-arg index="0" value="127.0.0.1"/>
<constructor-arg index="1" value="canal"/>
<constructor-arg index="2" value="canal"/>
<constructor-arg index="3" value="true"/>
<constructor-arg index="4" value="tsdb"/>
<constructor-arg index="5" value="jdbc:mysql://127.0.0.1:3306/tsdb"/>
<constructor-arg index="6" value="com.mysql.jdbc.Driver"/>
</bean>
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="dataSource"/>
</bean>
<bean id="txTemplate" class="org.springframework.transaction.support.TransactionTemplate">
<property name="transactionManager" ref="transactionManager"></property>
<property name="propagationBehaviorName" value="PROPAGATION_REQUIRED"></property>
</bean>
<bean id="sqlMapClient" class="org.springframework.orm.ibatis.SqlMapClientFactoryBean">
<property name="dataSource" ref="dataSource"/>
<property name="configLocation" value="classpath:sqlmap-config.xml"/>
</bean>
<bean id="MetaHistoryDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="MetaSnapshotDAO" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO">
<property name="sqlMapClient" ref="sqlMapClient"/>
</bean>
<bean id="PersistTableMeta" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager"></bean>
</beans>

23
parse/src/test/resources/ddl/create.sql Normal file
View File

@@ -0,0 +1,23 @@
CREATE TABLE `test` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',
`c_tinyint` tinyint(4) DEFAULT '1' COMMENT 'tinyint',
`c_smallint` smallint(6) DEFAULT 0 COMMENT 'smallint',
`c_mediumint` mediumint(9) DEFAULT NULL COMMENT 'mediumint',
`c_int` int(11) DEFAULT NULL COMMENT 'int',
`c_bigint` bigint(20) DEFAULT NULL COMMENT 'bigint',
`c_decimal` decimal(10,3) DEFAULT NULL COMMENT 'decimal',
`c_date` date DEFAULT '0000-00-00' COMMENT 'date',
`c_datetime` datetime DEFAULT '0000-00-00 00:00:00' COMMENT 'datetime',
`c_timestamp` timestamp NULL DEFAULT NULL COMMENT 'timestamp',
`c_time` time DEFAULT NULL COMMENT 'time',
`c_char` char(10) DEFAULT NULL COMMENT 'char',
`c_varchar` varchar(10) DEFAULT 'hello' COMMENT 'varchar',
`c_blob` blob COMMENT 'blob',
`c_text` text COMMENT 'text',
`c_mediumtext` mediumtext COMMENT 'mediumtext',
`c_longblob` longblob COMMENT 'longblob',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_a` (`c_tinyint`),
KEY `k_b` (`c_smallint`),
KEY `k_c` (`c_mediumint`,`c_int`)
) ENGINE=InnoDB AUTO_INCREMENT=1769503 DEFAULT CHARSET=utf8mb4 COMMENT='10000000';