From 7857df970125ba9de8e47af48dddd03dc2a9e573 Mon Sep 17 00:00:00 2001 From: agapple Date: Sat, 17 Apr 2021 10:49:17 +0800 Subject: [PATCH] add sql parser testcase --- .../adapter/es/core/config/SqlParser.java | 3 +- .../canal/example/SimpleCanalClientTest.java | 12 +-- .../inbound/mysql/tsdb/DatabaseTableMeta.java | 11 +-- .../inbound/mysql/tsdb/MemoryTableMeta.java | 9 ++- .../inbound/mysql/tsdb/FastsqlSchemaTest.java | 75 +++++++++++++++---- .../mysql/tsdb/MemoryTableMeta_DDL_Test.java | 22 +++++- .../tsdb/TableMetaManagerBuilderTest.java | 1 + .../resources/ddl/ddl_create_if_not_exist.sql | 12 +++ 8 files changed, 112 insertions(+), 33 deletions(-) create mode 100644 parse/src/test/resources/ddl/ddl_create_if_not_exist.sql diff --git a/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java b/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java index ed583971..12dce4fa 100644 --- a/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java +++ b/client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java @@ -230,8 +230,7 @@ public class SqlParser { } FieldItem rightFieldItem = new FieldItem(); visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem); - if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() - || rightFieldItem.isBinaryOp()) { + if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() || rightFieldItem.isBinaryOp()) { throw new UnsupportedOperationException("Unsupported for complex of on-condition"); } tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem)); diff --git a/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java index 0603d32c..21a25e9b 100644 --- a/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java +++ b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java @@ -15,17 +15,17 @@ import com.alibaba.otter.canal.common.utils.AddressUtils; public class SimpleCanalClientTest extends AbstractCanalClientTest { public SimpleCanalClientTest(String destination){ - super(destination); - } + super(destination); + } public static void main(String args[]) { // 根据ip，直接创建链接，无HA的功能 String destination = "example"; String ip = AddressUtils.getHostIp(); - CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1", 11111), - destination, - "canal", - "canal"); + CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111), + destination, + "canal", + "canal"); final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination); clientTest.setConnector(connector); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index 0488dbfc..8dd6ea8e 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -445,7 +445,7 @@ public class DatabaseTableMeta implements TableMetaTSDB { // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之 if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) { continue; - } else if (rollbackPosition.getServerId() == snapshotPosition.getServerId() + } else if (rollbackPosition.getServerId().equals(snapshotPosition.getServerId()) && snapshotPosition.compareTo(rollbackPosition) > 0) { continue; } @@ -472,14 +472,7 @@ public class DatabaseTableMeta implements TableMetaTSDB { private String getFullName(String schema, String table) { StringBuilder builder = new StringBuilder(); - return builder.append('`') - .append(structureSchema(schema)) - .append('`') - .append('.') - .append('`') - .append(table) - .append('`') - .toString(); + return builder.append(structureSchema(schema)).append('.').append('`').append(table).append('`').toString(); } public static boolean compareTableMeta(TableMeta source, TableMeta target) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java index ca250f13..55aa702d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -69,7 +69,7 @@ public class MemoryTableMeta implements TableMetaTSDB { tableMetas.clear(); synchronized (this) { if (StringUtils.isNotEmpty(schema)) { - repository.setDefaultSchema(schema); + repository.setDefaultSchema(structureSchema(schema)); } try { @@ -157,6 +157,13 @@ public class MemoryTableMeta implements TableMetaTSDB { return schemaDdls; } + private String structureSchema(String schema) { + if (schema.startsWith("`") && schema.endsWith("`")) { + return schema; + } + return "`" + schema + "`"; + } + private TableMeta parse(SQLCreateTableStatement statement) { int size = statement.getTableElementList().size(); if (size > 0) { diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java index c0913554..6888a55e 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java @@ -1,29 +1,76 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; -import java.io.FileNotFoundException; -import java.io.IOException; - +import org.junit.Assert; import org.junit.Test; import com.alibaba.druid.sql.repository.SchemaObject; import com.alibaba.druid.sql.repository.SchemaRepository; import com.alibaba.druid.util.JdbcConstants; -/** - * @author agapple 2018年6月7日 下午5:36:13 - * @since 3.1.9 - */ public class FastsqlSchemaTest { @Test - public void testSimple() throws FileNotFoundException, IOException { + public void testSimple() throws Throwable { SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); - String sql = "create table quniya4(name varchar(255) null,value varchar(255) null,id int not null,constraint quniya4_pk primary key (id));" - + "alter table quniya4 modify id int not null first;"; - repository.console(sql); - + String sql1 = "CREATE TABLE `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, " + + "`key1` longtext NOT NULL COMMENT 'key1', `value1` longtext NOT NULL COMMENT 'value1', PRIMARY KEY (`id`) )" + + "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"; + String sql2 = " CREATE TABLE IF NOT EXISTS `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT," + + "`key1` longtext NOT NULL COMMENT 'key1',PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"; + repository.console(sql1); + repository.console(sql2); repository.setDefaultSchema("test"); - SchemaObject table = repository.findTable("quniya4"); - System.out.println(table.getStatement().toString()); + SchemaObject table = repository.findTable("table_x1"); + Assert.assertTrue(table.findColumn("value1") != null); + } + + @Test + public void test_block_format() throws Throwable { + SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); + String sql = " CREATE TABLE `parent` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT," + + "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`)" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED"; + repository.console(sql); + repository.setDefaultSchema("test"); + SchemaObject table = repository.findTable("parent"); + Assert.assertTrue(table.findColumn("id") != null); + } + + @Test + public void test_json_index() throws Throwable { + SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); + String sql = " CREATE TABLE `articles` ( `article_id` bigint NOT NULL AUTO_INCREMENT," + + " `tags` json DEFAULT NULL, PRIMARY KEY (`article_id`)," + + " KEY `articles_tags` ((cast(json_extract(`tags`,_utf8mb4'$[*]') as char(40) array)))" + + ") ENGINE=InnoDB AUTO_INCREMENT=1054 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; + repository.console(sql); + repository.setDefaultSchema("test"); + SchemaObject table = repository.findTable("articles"); + Assert.assertTrue(table.findColumn("article_id") != null); + } + + @Test + public void test_invisible() throws Throwable { + SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); + String sql = " CREATE TABLE `proposal_order_info` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT," + + "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`) , " + + "KEY `idx_create_time` (`created_at`) /*!80000 INVISIBLE */" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED"; + repository.console(sql); + repository.setDefaultSchema("test"); + SchemaObject table = repository.findTable("proposal_order_info"); + Assert.assertTrue(table.findColumn("id") != null); + } + + @Test + public void test_persistent() throws Throwable { + SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); + String sql = " create table example_vc_tbl( c1 int not null auto_increment primary key," + + "c2 varchar(70), vc1 int as (length(c2)) virtual," + + "DIM_SUM varchar(128) AS (MD5(UPPER(CONCAT(c2, c1)))) PERSISTENT)"; + repository.console(sql); + repository.setDefaultSchema("test"); + SchemaObject table = repository.findTable("proposal_order_info"); + Assert.assertTrue(table.findColumn("id") != null); } } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java index 8c2c6e8e..eb2d575d 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java @@ -78,7 +78,27 @@ public class MemoryTableMeta_DDL_Test { File create = new File(dummyFile.getParent() + "/ddl", "ddl_any.sql"); String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); memoryTableMeta.apply(null, "test", sql, null); - + + List tableNames = Lists.newArrayList(); + for (Schema schema : memoryTableMeta.getRepository().getSchemas()) { + tableNames.addAll(schema.showTables()); + } + + for (String table : tableNames) { + TableMeta sourceMeta = memoryTableMeta.find("test", table); + System.out.println(sourceMeta.toString()); + } + } + + @Test + public void test_create_if_not_exist() throws Throwable { + MemoryTableMeta memoryTableMeta = new MemoryTableMeta(); + URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); + File dummyFile = new File(url.getFile()); + File create = new File(dummyFile.getParent() + "/ddl", "ddl_create_if_not_exist.sql"); + String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); + memoryTableMeta.apply(null, "test", sql, null); + List tableNames = Lists.newArrayList(); for (Schema schema : memoryTableMeta.getRepository().getSchemas()) { tableNames.addAll(schema.showTables()); diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java index e49ee735..148da964 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java @@ -9,6 +9,7 @@ import org.springframework.util.Assert; * @since 1.0.25 */ public class TableMetaManagerBuilderTest { + @Ignore @Test public void testSimple() { diff --git a/parse/src/test/resources/ddl/ddl_create_if_not_exist.sql b/parse/src/test/resources/ddl/ddl_create_if_not_exist.sql new file mode 100644 index 00000000..a5ee7203 --- /dev/null +++ b/parse/src/test/resources/ddl/ddl_create_if_not_exist.sql @@ -0,0 +1,12 @@ +CREATE TABLE `table_x1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `key1` longtext NOT NULL COMMENT 'key1', + `value1` longtext NOT NULL COMMENT 'value1', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS `table_x1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `key1` longtext NOT NULL COMMENT 'key1', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file