chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

add sql parser testcase

Browse Source
This commit is contained in:
agapple
2021-04-17 10:49:17 +08:00
parent 64ea4ff222
commit 7857df9701
8 changed files with 112 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java
View File

@@ -230,8 +230,7 @@ public class SqlParser {
}
FieldItem rightFieldItem = new FieldItem();
visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
|| rightFieldItem.isBinaryOp()) {
if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() || rightFieldItem.isBinaryOp()) {
throw new UnsupportedOperationException("Unsupported for complex of on-condition");
}
tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));

2
example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java
View File

@@ -22,7 +22,7 @@ public class SimpleCanalClientTest extends AbstractCanalClientTest {
// 根据ip，直接创建链接，无HA的功能
String destination = "example";
String ip = AddressUtils.getHostIp();
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1", 11111),
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
destination,
"canal",
"canal");

11
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -445,7 +445,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
// 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
continue;
} else if (rollbackPosition.getServerId() == snapshotPosition.getServerId()
} else if (rollbackPosition.getServerId().equals(snapshotPosition.getServerId())
&& snapshotPosition.compareTo(rollbackPosition) > 0) {
continue;
}
@@ -472,14 +472,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private String getFullName(String schema, String table) {
StringBuilder builder = new StringBuilder();
return builder.append('`')
.append(structureSchema(schema))
.append('`')
.append('.')
.append('`')
.append(table)
.append('`')
.toString();
return builder.append(structureSchema(schema)).append('.').append('`').append(table).append('`').toString();
}
public static boolean compareTableMeta(TableMeta source, TableMeta target) {

9
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -69,7 +69,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
tableMetas.clear();
synchronized (this) {
if (StringUtils.isNotEmpty(schema)) {
repository.setDefaultSchema(schema);
repository.setDefaultSchema(structureSchema(schema));
}
try {
@@ -157,6 +157,13 @@ public class MemoryTableMeta implements TableMetaTSDB {
return schemaDdls;
}
private String structureSchema(String schema) {
if (schema.startsWith("`") && schema.endsWith("`")) {
return schema;
}
return "`" + schema + "`";
}
private TableMeta parse(SQLCreateTableStatement statement) {
int size = statement.getTableElementList().size();
if (size > 0) {

75
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java
View File

@@ -1,29 +1,76 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.druid.sql.repository.SchemaObject;
import com.alibaba.druid.sql.repository.SchemaRepository;
import com.alibaba.druid.util.JdbcConstants;
/**
* @author agapple 2018年6月7日 下午5:36:13
* @since 3.1.9
*/
public class FastsqlSchemaTest {
@Test
public void testSimple() throws FileNotFoundException, IOException {
public void testSimple() throws Throwable {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = "create table quniya4(name varchar(255) null,value varchar(255) null,id int not null,constraint quniya4_pk primary key (id));"
+ "alter table quniya4 modify id int not null first;";
repository.console(sql);
String sql1 = "CREATE TABLE `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, "
+ "`key1` longtext NOT NULL COMMENT 'key1', `value1` longtext NOT NULL COMMENT 'value1', PRIMARY KEY (`id`) )"
+ "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
String sql2 = " CREATE TABLE IF NOT EXISTS `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT,"
+ "`key1` longtext NOT NULL COMMENT 'key1',PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
repository.console(sql1);
repository.console(sql2);
repository.setDefaultSchema("test");
SchemaObject table = repository.findTable("quniya4");
System.out.println(table.getStatement().toString());
SchemaObject table = repository.findTable("table_x1");
Assert.assertTrue(table.findColumn("value1") != null);
}
@Test
public void test_block_format() throws Throwable {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = " CREATE TABLE `parent` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,"
+ "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`)"
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED";
repository.console(sql);
repository.setDefaultSchema("test");
SchemaObject table = repository.findTable("parent");
Assert.assertTrue(table.findColumn("id") != null);
}
@Test
public void test_json_index() throws Throwable {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = " CREATE TABLE `articles` ( `article_id` bigint NOT NULL AUTO_INCREMENT,"
+ " `tags` json DEFAULT NULL, PRIMARY KEY (`article_id`),"
+ " KEY `articles_tags` ((cast(json_extract(`tags`,_utf8mb4'$[*]') as char(40) array)))"
+ ") ENGINE=InnoDB AUTO_INCREMENT=1054 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci";
repository.console(sql);
repository.setDefaultSchema("test");
SchemaObject table = repository.findTable("articles");
Assert.assertTrue(table.findColumn("article_id") != null);
}
@Test
public void test_invisible() throws Throwable {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = " CREATE TABLE `proposal_order_info` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,"
+ "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`) , "
+ "KEY `idx_create_time` (`created_at`) /*!80000 INVISIBLE */"
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED";
repository.console(sql);
repository.setDefaultSchema("test");
SchemaObject table = repository.findTable("proposal_order_info");
Assert.assertTrue(table.findColumn("id") != null);
}
@Test
public void test_persistent() throws Throwable {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = " create table example_vc_tbl( c1 int not null auto_increment primary key,"
+ "c2 varchar(70), vc1 int as (length(c2)) virtual,"
+ "DIM_SUM varchar(128) AS (MD5(UPPER(CONCAT(c2, c1)))) PERSISTENT)";
repository.console(sql);
repository.setDefaultSchema("test");
SchemaObject table = repository.findTable("proposal_order_info");
Assert.assertTrue(table.findColumn("id") != null);
}
}

20
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java
View File

@@ -89,4 +89,24 @@ public class MemoryTableMeta_DDL_Test {
System.out.println(sourceMeta.toString());
}
}
@Test
public void test_create_if_not_exist() throws Throwable {
MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
File create = new File(dummyFile.getParent() + "/ddl", "ddl_create_if_not_exist.sql");
String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
memoryTableMeta.apply(null, "test", sql, null);
List<String> tableNames = Lists.newArrayList();
for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
tableNames.addAll(schema.showTables());
}
for (String table : tableNames) {
TableMeta sourceMeta = memoryTableMeta.find("test", table);
System.out.println(sourceMeta.toString());
}
}
}

1
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java
View File

@@ -9,6 +9,7 @@ import org.springframework.util.Assert;
* @since 1.0.25
*/
public class TableMetaManagerBuilderTest {
@Ignore
@Test
public void testSimple() {

12
parse/src/test/resources/ddl/ddl_create_if_not_exist.sql Normal file
View File

@@ -0,0 +1,12 @@
CREATE TABLE `table_x1` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`key1` longtext NOT NULL COMMENT 'key1',
`value1` longtext NOT NULL COMMENT 'value1',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE IF NOT EXISTS `table_x1` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`key1` longtext NOT NULL COMMENT 'key1',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;