chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #535 , use show create table to get tableMeta

Browse Source
This commit is contained in:
七锋
2018-03-12 16:27:58 +08:00
parent 0af789b1c7
commit 847ebf9bf3
2 changed files with 32 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
View File

@@ -2,19 +2,17 @@ package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.MemoryTableMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheBuilder;
@@ -77,43 +75,23 @@ public class TableMetaCache {
} }
private TableMeta getTableMetaByDB(String fullname) throws IOException { private TableMeta getTableMetaByDB(String fullname) throws IOException {
ResultSetPacket packet = connection.query("desc " + fullname); ResultSetPacket packet = connection.query("show create table " + fullname);
String[] names = StringUtils.split(fullname, "`.`"); String[] names = StringUtils.split(fullname, "`.`");
String schema = names[0]; String schema = names[0];
String table = names[1].substring(0, names[1].length()); String table = names[1].substring(0, names[1].length());
return new TableMeta(schema, table, parserTableMeta(packet)); return new TableMeta(schema, table, parserTableMeta(schema, table, packet));
} }
public static List<FieldMeta> parserTableMeta(ResultSetPacket packet) { public static List<FieldMeta> parserTableMeta(String schema, String table, ResultSetPacket packet) {
Map<String, Integer> nameMaps = new HashMap<String, Integer>(6, 1f); if (packet.getFieldValues().size() > 1) {
String createDDL = packet.getFieldValues().get(1);
int index = 0; MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
for (FieldPacket fieldPacket : packet.getFieldDescriptors()) { memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
nameMaps.put(fieldPacket.getOriginalName(), index++); TableMeta tableMeta = memoryTableMeta.find(schema, table);
return tableMeta.getFields();
} else {
return new ArrayList<FieldMeta>();
} }
int size = packet.getFieldDescriptors().size();
int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size();
List<FieldMeta> result = new ArrayList<FieldMeta>();
for (int i = 0; i < count; i++) {
FieldMeta meta = new FieldMeta();
// 做一个优化，使用String.intern()，共享String对象，减少内存使用
meta.setColumnName(packet.getFieldValues().get(nameMaps.get(COLUMN_NAME) + i * size).intern());
meta.setColumnType(packet.getFieldValues().get(nameMaps.get(COLUMN_TYPE) + i * size));
meta.setNullable(StringUtils.equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(IS_NULLABLE) + i
* size),
"YES"));
meta.setKey("PRI".equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size)));
meta.setUnique("UNI".equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size)));
// 特殊处理引号
meta.setDefaultValue(DruidDdlParser.unescapeQuotaName(packet.getFieldValues()
.get(nameMaps.get(COLUMN_DEFAULT) + i * size)));
meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size));
result.add(meta);
}
return result;
} }
public TableMeta getTableMeta(String schema, String table) { public TableMeta getTableMeta(String schema, String table) {

20
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -44,10 +44,10 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
*/ */
public class DatabaseTableMeta implements TableMetaTSDB { public class DatabaseTableMeta implements TableMetaTSDB {
public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class); private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
private static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
private String destination; private String destination;
private MemoryTableMeta memoryTableMeta; private MemoryTableMeta memoryTableMeta;
private MysqlConnection connection; // 查询meta信息的链接 private MysqlConnection connection; // 查询meta信息的链接
@@ -291,9 +291,13 @@ public class DatabaseTableMeta implements TableMetaTSDB {
TableMeta tableMetaFromDB = new TableMeta(); TableMeta tableMetaFromDB = new TableMeta();
tableMetaFromDB.setSchema(schema); tableMetaFromDB.setSchema(schema);
tableMetaFromDB.setTable(table); tableMetaFromDB.setTable(table);
String createDDL = null;
try { try {
ResultSetPacket packet = connection.query("desc " + getFullName(schema, table)); ResultSetPacket packet = connection.query("show create table " + getFullName(schema, table));
tableMetaFromDB.setFields(TableMetaCache.parserTableMeta(packet)); if (packet.getFieldValues().size() > 1) {
createDDL = packet.getFieldValues().get(1);
tableMetaFromDB.setFields(TableMetaCache.parserTableMeta(schema, table, packet));
}
} catch (IOException e) { } catch (IOException e) {
if (e.getMessage().contains("errorNumber=1146")) { if (e.getMessage().contains("errorNumber=1146")) {
logger.error("table not exist in db , pls check :" + getFullName(schema, table) + " , mem : " logger.error("table not exist in db , pls check :" + getFullName(schema, table) + " , mem : "
@@ -305,16 +309,6 @@ public class DatabaseTableMeta implements TableMetaTSDB {
boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB); boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB);
if (!result) { if (!result) {
String createDDL = null;
try {
ResultSetPacket packet = connection.query("show create table " + getFullName(schema, table));
if (packet.getFieldValues().size() > 1) {
createDDL = packet.getFieldValues().get(1);
}
} catch (IOException e) {
// ignore
}
logger.error("pls submit github issue, show create table ddl:" + createDDL + " , compare failed . \n db : " logger.error("pls submit github issue, show create table ddl:" + createDDL + " , compare failed . \n db : "
+ tableMetaFromDB + " \n mem : " + tableMetaFromMem); + tableMetaFromDB + " \n mem : " + tableMetaFromMem);
} }