chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #136 , support rds no primary key && mysql.ha_health_check

Browse Source
This commit is contained in:
agapple
2016-11-17 17:52:52 +08:00
parent ad2a96b3f1
commit 96dca0b9e6
4 changed files with 84 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java
View File

@@ -104,7 +104,7 @@ public class MysqlConnector {
executor.update("KILL CONNECTION " + connectionId);
} catch (Exception e) {
// 忽略具体异常
logger.warn("KILL DUMP " + connectionId + " failure:" + ExceptionUtils.getStackTrace(e));
logger.info("KILL DUMP " + connectionId + " failure:" + ExceptionUtils.getStackTrace(e));
} finally {
if (connector != null) {
connector.disconnect();

12
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.parse.inbound;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -44,6 +45,17 @@ public class TableMeta {
this.fileds = fileds;
}
public List<FieldMeta> getPrimaryFields() {
List<FieldMeta> primarys = new ArrayList<TableMeta.FieldMeta>();
for (FieldMeta meta : fileds) {
if (meta.isKey()) {
primarys.add(meta);
}
}
return primarys;
}
public static class FieldMeta {
private String columnName;

40
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -326,6 +326,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return null;
}
if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
// 忽略rds模式的mysql.ha_health_check心跳数据
return null;
}
EventType eventType = null;
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
@@ -402,12 +407,25 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols,
boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
final int columnCnt = event.getTable().getColumnCnt();
final ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
int columnCnt = event.getTable().getColumnCnt();
ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
boolean tableError = false;
// check table fileds count，只能处理加字段
boolean existRDSNoPrimaryKey = false;
if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
if (tableMetaCache.isOnRDS()) {
// 特殊处理下RDS的场景
List<FieldMeta> primaryKeys = tableMeta.getPrimaryFields();
if (primaryKeys == null || primaryKeys.isEmpty()) {
if (columnInfo.length == tableMeta.getFileds().size() + 1
&& columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) {
existRDSNoPrimaryKey = true;
}
}
}
if (!existRDSNoPrimaryKey) {
// online ddl增加字段操作步骤：
// 1. 新增一张临时表，将需要做ddl表的数据全量导入
// 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
@@ -427,10 +445,14 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
tableError = true;
if (!filterTableError) {
throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + ","
+ columnInfo.length + " vs " + tableMeta.getFileds().size());
throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName()
+ "," + columnInfo.length + " vs " + tableMeta.getFileds().size());
}
}
} else {
logger.warn("[" + event.getTable().getDbName() + "." + event.getTable().getTableName()
+ "] is no primary key , skip alibaba_rds_row_id column");
}
}
for (int i = 0; i < columnCnt; i++) {
@@ -440,6 +462,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
continue;
}
if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
// 不解析最后一列
buffer.nextValue(info.type, info.meta, false);
continue;
}
Column.Builder columnBuilder = Column.newBuilder();
FieldMeta fieldMeta = null;
@@ -465,6 +493,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
}
buffer.nextValue(info.type, info.meta, isBinary);
if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
// 不解析最后一列
continue;
}
int javaType = buffer.getJavaType();
if (buffer.isNull()) {

17
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
View File

@@ -32,6 +32,7 @@ public class TableMetaCache {
public static final String COLUMN_DEFAULT = "COLUMN_DEFAULT";
public static final String EXTRA = "EXTRA";
private MysqlConnection connection;
private boolean isOnRDS = false;
// 第一层tableId,第二层schema.table,解决tableId重复，对应多张表
private Map<String, TableMeta> tableMetaCache;
@@ -56,6 +57,13 @@ public class TableMetaCache {
});
try {
ResultSetPacket packet = connection.query("show global variables like 'rds\\_%'");
if (packet.getFieldValues().size() > 0) {
isOnRDS = true;
}
} catch (IOException e) {
}
}
public TableMeta getTableMeta(String schema, String table) {
@@ -136,4 +144,13 @@ public class TableMetaCache {
.append('`')
.toString();
}
public boolean isOnRDS() {
return isOnRDS;
}
public void setOnRDS(boolean isOnRDS) {
this.isOnRDS = isOnRDS;
}
}