add sql parser testcase

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java

@@ -230,8 +230,7 @@ public class SqlParser {
             }
             FieldItem rightFieldItem = new FieldItem();
             visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
-            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
+            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() || rightFieldItem.isBinaryOp()) {
-                || rightFieldItem.isBinaryOp()) {
                 throw new UnsupportedOperationException("Unsupported for complex of on-condition");
             }
             tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));

example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java

@@ -15,17 +15,17 @@ import com.alibaba.otter.canal.common.utils.AddressUtils;
 public class SimpleCanalClientTest extends AbstractCanalClientTest {
 
     public SimpleCanalClientTest(String destination){
-                super(destination);
+        super(destination);
-            }
+    }
 
     public static void main(String args[]) {
         // 根据ip，直接创建链接，无HA的功能
         String destination = "example";
         String ip = AddressUtils.getHostIp();
-        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1", 11111),
+        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
-                destination,
+            destination,
-                "canal",
+            "canal",
-                "canal");
+            "canal");
 
         final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
         clientTest.setConnector(connector);

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java

@@ -445,7 +445,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
                 // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
                 if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
                     continue;
-                } else if (rollbackPosition.getServerId() == snapshotPosition.getServerId()
+                } else if (rollbackPosition.getServerId().equals(snapshotPosition.getServerId())
                            && snapshotPosition.compareTo(rollbackPosition) > 0) {
                     continue;
                 }
@@ -472,14 +472,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
 
     private String getFullName(String schema, String table) {
         StringBuilder builder = new StringBuilder();
-        return builder.append('`')
+        return builder.append(structureSchema(schema)).append('.').append('`').append(table).append('`').toString();
-            .append(structureSchema(schema))
-            .append('`')
-            .append('.')
-            .append('`')
-            .append(table)
-            .append('`')
-            .toString();
     }
 
     public static boolean compareTableMeta(TableMeta source, TableMeta target) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java

@@ -69,7 +69,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
         tableMetas.clear();
         synchronized (this) {
             if (StringUtils.isNotEmpty(schema)) {
-                repository.setDefaultSchema(schema);
+                repository.setDefaultSchema(structureSchema(schema));
             }
 
             try {
@@ -157,6 +157,13 @@ public class MemoryTableMeta implements TableMetaTSDB {
         return schemaDdls;
     }
 
+    private String structureSchema(String schema) {
+        if (schema.startsWith("`") && schema.endsWith("`")) {
+            return schema;
+        }
+        return "`" + schema + "`";
+    }
+
     private TableMeta parse(SQLCreateTableStatement statement) {
         int size = statement.getTableElementList().size();
         if (size > 0) {

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java

@@ -1,29 +1,76 @@
 package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
 
-import java.io.FileNotFoundException;
+import org.junit.Assert;
-import java.io.IOException;
-
 import org.junit.Test;
 
 import com.alibaba.druid.sql.repository.SchemaObject;
 import com.alibaba.druid.sql.repository.SchemaRepository;
 import com.alibaba.druid.util.JdbcConstants;
 
-/**
- * @author agapple 2018年6月7日 下午5:36:13
- * @since 3.1.9
- */
 public class FastsqlSchemaTest {
 
     @Test
-    public void testSimple() throws FileNotFoundException, IOException {
+    public void testSimple() throws Throwable {
         SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
-        String sql = "create table quniya4(name varchar(255) null,value varchar(255) null,id int not null,constraint quniya4_pk primary key (id));"
+        String sql1 = "CREATE TABLE `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, "
-                     + "alter table quniya4 modify id int not null first;";
+                      + "`key1` longtext NOT NULL COMMENT 'key1', `value1` longtext NOT NULL COMMENT 'value1', PRIMARY KEY (`id`) )"
-        repository.console(sql);
+                      + "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
-
+        String sql2 = " CREATE TABLE IF NOT EXISTS `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT,"
+                      + "`key1` longtext NOT NULL COMMENT 'key1',PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
+        repository.console(sql1);
+        repository.console(sql2);
         repository.setDefaultSchema("test");
-        SchemaObject table = repository.findTable("quniya4");
+        SchemaObject table = repository.findTable("table_x1");
-        System.out.println(table.getStatement().toString());
+        Assert.assertTrue(table.findColumn("value1") != null);
+    }
+
+    @Test
+    public void test_block_format() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " CREATE TABLE `parent` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,"
+                     + "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`)"
+                     + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("parent");
+        Assert.assertTrue(table.findColumn("id") != null);
+    }
+
+    @Test
+    public void test_json_index() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " CREATE TABLE `articles` ( `article_id` bigint NOT NULL AUTO_INCREMENT,"
+                     + " `tags` json DEFAULT NULL, PRIMARY KEY (`article_id`),"
+                     + " KEY `articles_tags` ((cast(json_extract(`tags`,_utf8mb4'$[*]') as char(40) array)))"
+                     + ") ENGINE=InnoDB AUTO_INCREMENT=1054 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("articles");
+        Assert.assertTrue(table.findColumn("article_id") != null);
+    }
+
+    @Test
+    public void test_invisible() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " CREATE TABLE `proposal_order_info` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,"
+                     + "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`) , "
+                     + "KEY `idx_create_time` (`created_at`) /*!80000 INVISIBLE */"
+                     + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("proposal_order_info");
+        Assert.assertTrue(table.findColumn("id") != null);
+    }
+
+    @Test
+    public void test_persistent() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " create table example_vc_tbl( c1 int not null auto_increment primary key,"
+                     + "c2 varchar(70), vc1 int as (length(c2)) virtual,"
+                     + "DIM_SUM varchar(128) AS (MD5(UPPER(CONCAT(c2, c1)))) PERSISTENT)";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("proposal_order_info");
+        Assert.assertTrue(table.findColumn("id") != null);
     }
 }

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java

@@ -78,7 +78,27 @@ public class MemoryTableMeta_DDL_Test {
         File create = new File(dummyFile.getParent() + "/ddl", "ddl_any.sql");
         String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
         memoryTableMeta.apply(null, "test", sql, null);
-        
+
+        List<String> tableNames = Lists.newArrayList();
+        for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
+            tableNames.addAll(schema.showTables());
+        }
+
+        for (String table : tableNames) {
+            TableMeta sourceMeta = memoryTableMeta.find("test", table);
+            System.out.println(sourceMeta.toString());
+        }
+    }
+
+    @Test
+    public void test_create_if_not_exist() throws Throwable {
+        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
+        URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
+        File dummyFile = new File(url.getFile());
+        File create = new File(dummyFile.getParent() + "/ddl", "ddl_create_if_not_exist.sql");
+        String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
+        memoryTableMeta.apply(null, "test", sql, null);
+
         List<String> tableNames = Lists.newArrayList();
         for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
             tableNames.addAll(schema.showTables());

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java

@@ -9,6 +9,7 @@ import org.springframework.util.Assert;
  * @since 1.0.25
  */
 public class TableMetaManagerBuilderTest {
+
     @Ignore
     @Test
     public void testSimple() {

parse/src/test/resources/ddl/ddl_create_if_not_exist.sql

@@ -0,0 +1,12 @@
+CREATE TABLE `table_x1` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `key1` longtext NOT NULL COMMENT 'key1',
+    `value1` longtext NOT NULL COMMENT 'value1',
+    PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+CREATE TABLE IF NOT EXISTS `table_x1` (
+    `id` bigint(20) NOT NULL AUTO_INCREMENT,
+    `key1` longtext NOT NULL COMMENT 'key1',
+    PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;