chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/ziyang_migrate_parser

Browse Source
This commit is contained in:
agapple
2024-05-22 21:05:31 +08:00
committed by GitHub
parent a6d73ad16b aa05d8e2a0
commit a98549ee0d
19 changed files with 438 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java
View File

@@ -24,7 +24,7 @@ public class TableMeta {
private String schema;
private String table;
private List<FieldMeta> fields = new ArrayList<>();
private String ddl; // 表结构的DDL语句
private String ddl; // 表结构的DDL语句
public TableMeta(){
@@ -74,6 +74,22 @@ public class TableMeta {
throw new RuntimeException("unknow column : " + name);
}
/**
* 尝试基于列名查找一下Filed信息,找不到时返回为null
*
* @param name
* @return
*/
public FieldMeta tryGetFieldMetaByName(String name) {
for (FieldMeta meta : fields) {
if (meta.getColumnName().equalsIgnoreCase(name)) {
return meta;
}
}
return null;
}
public List<FieldMeta> getPrimaryFields() {
List<FieldMeta> primarys = new ArrayList<>();
for (FieldMeta meta : fields) {

86
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -13,7 +13,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
import com.alibaba.polardbx.druid.sql.repository.Schema;
import org.apache.commons.beanutils.BeanUtils;
@@ -26,6 +25,7 @@ import org.slf4j.MDC;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
@@ -247,30 +247,47 @@ public class DatabaseTableMeta implements TableMetaTSDB {
}
private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) {
Map<String, String> content = new HashMap<>();
content.put("destination", destination);
content.put("binlogFile", position.getJournalName());
content.put("binlogOffest", String.valueOf(position.getPosition()));
content.put("binlogMasterId", String.valueOf(position.getServerId()));
content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
content.put("useSchema", schema);
if (content.isEmpty()) {
throw new RuntimeException("apply failed caused by content is empty in applyHistoryToDB");
}
// 待补充
// Map<String, String> content = new HashMap<>();
// content.put("destination", destination);
// content.put("binlogFile", position.getJournalName());
// content.put("binlogOffest", String.valueOf(position.getPosition()));
// content.put("binlogMasterId", String.valueOf(position.getServerId()));
// content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
// content.put("useSchema", schema);
//
// if (content.isEmpty()) {
// throw new RuntimeException("apply failed caused by content is empty in
// applyHistoryToDB");
// }
// // 待补充
// List<DdlResult> ddlResults = DruidDdlParser.parse(ddl, schema);
// if (ddlResults.size() > 0) {
// DdlResult ddlResult = ddlResults.get(0);
// content.put("sqlSchema", ddlResult.getSchemaName());
// content.put("sqlTable", ddlResult.getTableName());
// content.put("sqlType", ddlResult.getType().name());
// content.put("sqlText", ddl);
// content.put("extra", extra);
// }
// BeanUtils.populate(metaDO, content);
MetaHistoryDO metaDO = new MetaHistoryDO();
metaDO.setDestination(destination);
metaDO.setBinlogFile(position.getJournalName());
metaDO.setBinlogOffest(position.getPosition());
metaDO.setBinlogMasterId(String.valueOf(position.getServerId()));
metaDO.setBinlogTimestamp(position.getTimestamp());
metaDO.setUseSchema(schema);
List<DdlResult> ddlResults = DruidDdlParser.parse(ddl, schema);
if (ddlResults.size() > 0) {
DdlResult ddlResult = ddlResults.get(0);
content.put("sqlSchema", ddlResult.getSchemaName());
content.put("sqlTable", ddlResult.getTableName());
content.put("sqlType", ddlResult.getType().name());
content.put("sqlText", ddl);
content.put("extra", extra);
metaDO.setSqlSchema(ddlResult.getSchemaName());
metaDO.setSqlTable(ddlResult.getTableName());
metaDO.setSqlType(ddlResult.getType().name());
metaDO.setSqlText(ddl);
metaDO.setExtra(extra);
}
MetaHistoryDO metaDO = new MetaHistoryDO();
try {
BeanUtils.populate(metaDO, content);
// 会建立唯一约束,解决:
// 1. 重复的binlog file+offest
// 2. 重复的masterId+timestamp
@@ -328,20 +345,27 @@ public class DatabaseTableMeta implements TableMetaTSDB {
}
if (compareAll) {
Map<String, String> content = new HashMap<>();
content.put("destination", destination);
content.put("binlogFile", position.getJournalName());
content.put("binlogOffest", String.valueOf(position.getPosition()));
content.put("binlogMasterId", String.valueOf(position.getServerId()));
content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
content.put("data", JSON.toJSONString(schemaDdls));
if (content.isEmpty()) {
throw new RuntimeException("apply failed caused by content is empty in applySnapshotToDB");
}
// Map<String, String> content = new HashMap<>();
// content.put("destination", destination);
// content.put("binlogFile", position.getJournalName());
// content.put("binlogOffest", String.valueOf(position.getPosition()));
// content.put("binlogMasterId", String.valueOf(position.getServerId()));
// content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
// content.put("data", JSON.toJSONString(schemaDdls));
// if (content.isEmpty()) {
// throw new RuntimeException("apply failed caused by content is empty in
// applySnapshotToDB");
// }
// BeanUtils.populate(snapshotDO, content);
MetaSnapshotDO snapshotDO = new MetaSnapshotDO();
snapshotDO.setDestination(destination);
snapshotDO.setBinlogFile(position.getJournalName());
snapshotDO.setBinlogOffest(position.getPosition());
snapshotDO.setBinlogMasterId(String.valueOf(position.getServerId()));
snapshotDO.setBinlogTimestamp(position.getTimestamp());
snapshotDO.setExtra(JSON.toJSONString(schemaDdls));
try {
BeanUtils.populate(snapshotDO, content);
metaSnapshotDAO.insert(snapshotDO);
} catch (Throwable e) {
if (isUkDuplicateException(e)) {

51
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -1,9 +1,6 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.polardbx.druid.DbType;
@@ -38,7 +35,6 @@ import com.alibaba.polardbx.druid.util.JdbcConstants;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser;
@@ -272,11 +268,50 @@ public class MemoryTableMeta implements TableMetaTSDB {
} else if (element instanceof MySqlUnique) {
MySqlUnique column = (MySqlUnique) element;
List<SQLSelectOrderByItem> uks = column.getColumns();
// https://github.com/alibaba/canal/issues/5094
// 处理一下函数索引
List<String> columnNames = new ArrayList<String>();
for (SQLSelectOrderByItem uk : uks) {
String name = getSqlName(uk.getExpr());
FieldMeta field = tableMeta.getFieldMetaByName(name);
field.setUnique(true);
SQLExpr sqlName = uk.getExpr();
columnNames.addAll(getIndexColumnNames(sqlName));
}
// uniqe打标
for (String name : columnNames) {
FieldMeta field = tableMeta.tryGetFieldMetaByName(name);
if (field != null) {
field.setUnique(true);
}
}
}
}
private List<String> getIndexColumnNames(SQLExpr expr) {
if (expr instanceof SQLMethodInvokeExpr) {
// 需要递归处理下函数索引, 尽可能收集一下列名
// 常见的case:
// 1. upper(col)
// 2. left(upper(col) , 10)
// 3. col(10)
List<SQLExpr> indexExpres = ((SQLMethodInvokeExpr) expr).getArguments();
List<String> columnNames = new ArrayList<String>();
// 加上当前列
columnNames.add(getSqlName(expr));
// 处理函数索引列
for (SQLExpr exArgs : indexExpres) {
if (exArgs instanceof SQLMethodInvokeExpr) {
// 加上当前列
columnNames.add(getSqlName(exArgs));
columnNames.addAll(getIndexColumnNames(exArgs));
} else {
String columnName = getSqlName(exArgs);
columnNames.add(columnName);
}
}
return columnNames;
} else {
String columnName = getSqlName(expr);
return Arrays.asList(columnName);
}
}

13
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java
View File

@@ -327,4 +327,17 @@ public class FastsqlSchemaTest {
System.out.println(data.toString());
}
@Test
public void test_function_index () throws Throwable {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
repository.setDefaultSchema("test");
String sql = "CREATE TABLE test1 (\n" + " id INT AUTO_INCREMENT PRIMARY KEY,\n"
+ " owner_id INT NOT NULL,\n" + " code VARCHAR(100) NOT NULL,\n"
+ " UNIQUE KEY uk_owner_id_upper_code (owner_id, (upper(code)))\n" + ");";
repository.console(sql);
SchemaObject table = repository.findTable("test1");
Assert.assertTrue(table != null);
}
}

25
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java
View File

@@ -6,6 +6,9 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.repository.SchemaObject;
import com.alibaba.druid.sql.repository.SchemaRepository;
import com.alibaba.druid.util.JdbcConstants;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
@@ -109,4 +112,26 @@ public class MemoryTableMeta_DDL_Test {
System.out.println(sourceMeta.toString());
}
}
@Test
public void test_function_index () throws Throwable {
MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
File dummyFile = new File(url.getFile());
File create = new File(dummyFile.getParent() + "/ddl", "ddl_create_function_index.sql");
String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
memoryTableMeta.apply(null, "test", sql, null);
List<String> tableNames = new ArrayList<>();
for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
tableNames.addAll(schema.showTables());
}
for (String table : tableNames) {
TableMeta sourceMeta = memoryTableMeta.find("test", table);
TableMeta.FieldMeta field = sourceMeta.getFieldMetaByName("code");
System.out.println(sourceMeta.toString());
Assert.assertTrue(field.isUnique());
}
}
}

1
parse/src/test/resources/ddl/ddl_create_function_index.sql Normal file
View File

@@ -0,0 +1 @@
CREATE TABLE function_index_test (