chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge code and fixed issue #2755 #2756 #2735 #2714 #2711 #2710 #2616 #2602 #2585 #2565 #2935 #2434 #2623 #2418 #2387

Browse Source
This commit is contained in:
agapple
2020-08-22 13:07:24 +08:00
parent 90bf9e6de5
commit b79af46054
66 changed files with 459 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
View File

@@ -50,7 +50,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
}
convert.setFieldFilterMap(getFieldFilterMap());
convert.setFieldBlackFilterMap(getFieldBlackFilterMap());
@@ -93,13 +93,13 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
}
}
}
@Override
public void setFieldFilter(String fieldFilter) {
super.setFieldFilter(fieldFilter);
// 触发一下filter变更
if (binlogParser instanceof LogEventConvert) {
super.setFieldFilter(fieldFilter);
// 触发一下filter变更
if (binlogParser instanceof LogEventConvert) {
((LogEventConvert) binlogParser).setFieldFilterMap(getFieldFilterMap());
}
@@ -107,13 +107,13 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
((DatabaseTableMeta) tableMetaTSDB).setFieldFilterMap(getFieldFilterMap());
}
}
@Override
public void setFieldBlackFilter(String fieldBlackFilter) {
super.setFieldBlackFilter(fieldBlackFilter);
// 触发一下filter变更
if (binlogParser instanceof LogEventConvert) {
super.setFieldBlackFilter(fieldBlackFilter);
// 触发一下filter变更
if (binlogParser instanceof LogEventConvert) {
((LogEventConvert) binlogParser).setFieldBlackFilterMap(getFieldBlackFilterMap());
}
@@ -184,11 +184,15 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
this.connectionCharsetNumber = connectionCharsetNumber;
}
public void setConnectionCharset(Charset connectionCharset) {
public void setConnectionCharsetStd(Charset connectionCharset) {
this.connectionCharset = connectionCharset;
}
public void setConnectionCharset(String connectionCharset) {
if ("UTF8MB4".equalsIgnoreCase(connectionCharset)) {
connectionCharset = "UTF-8";
}
this.connectionCharset = Charset.forName(connectionCharset);
}

6
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -393,13 +393,13 @@ public class MysqlConnection implements ErosaConnection {
logger.warn("update wait_timeout failed", e);
}
try {
update("set net_write_timeout=1800");
update("set net_write_timeout=7200");
} catch (Exception e) {
logger.warn("update net_write_timeout failed", e);
}
try {
update("set net_read_timeout=1800");
update("set net_read_timeout=7200");
} catch (Exception e) {
logger.warn("update net_read_timeout failed", e);
}
@@ -522,7 +522,7 @@ public class MysqlConnection implements ErosaConnection {
rs = query("select @@global.binlog_checksum");
List<String> columnValues = rs.getFieldValues();
if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0) != null
&& "CRC32".equals(columnValues.get(0).toUpperCase())) {
&& columnValues.get(0).toUpperCase().equals("CRC32")) {
binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
} else {
binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;

97
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -20,7 +20,6 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
@@ -59,6 +58,7 @@ import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
/**
* 基于{@linkplain LogEvent}转化为Entry对象的处理
@@ -88,8 +88,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时
private volatile AviaterRegexFilter nameBlackFilter;
private Map<String, List<String>> fieldFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<String, List<String>>();
private TableMetaCache tableMetaCache;
private Charset charset = Charset.defaultCharset();
@@ -119,6 +119,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
case LogEvent.XID_EVENT:
return parseXidEvent((XidLogEvent) logEvent);
case LogEvent.TABLE_MAP_EVENT:
parseTableMapEvent((TableMapLogEvent) logEvent);
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:
@@ -144,6 +145,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return parseGTIDLogEvent((GtidLogEvent) logEvent);
case LogEvent.HEARTBEAT_LOG_EVENT:
return parseHeartbeatLogEvent((HeartbeatLogEvent) logEvent);
default:
break;
}
@@ -275,7 +277,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
Header header = createHeader(event.getHeader(), schemaName, tableName, type);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
if (type != EventType.QUERY && !isDml) {
if (type == EventType.QUERY && !isDml) {
rowChangeBuider.setIsDdl(true);
}
rowChangeBuider.setSql(queryString);
@@ -491,6 +493,18 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return parseRowsEvent(event, null);
}
public void parseTableMapEvent(TableMapLogEvent event) {
try {
String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset.name());
event.setDbname(charsetDbName);
String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset.name());
event.setTblname(charsetTbName);
} catch (UnsupportedEncodingException e) {
throw new CanalParseException(e);
}
}
public Entry parseRowsEvent(RowsLogEvent event, TableMeta tableMeta) {
if (filterRows) {
return null;
@@ -588,15 +602,15 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
boolean tableError = false;
// check table fileds count，只能处理加字段
boolean existRDSNoPrimaryKey = false;
//获取字段过滤条件
// 获取字段过滤条件
List<String> fieldList = null;
List<String> blackFieldList = null;
if (tableMeta != null) {
fieldList = fieldFilterMap.get(tableMeta.getFullName().toUpperCase());
blackFieldList = fieldBlackFilterMap.get(tableMeta.getFullName().toUpperCase());
fieldList = fieldFilterMap.get(tableMeta.getFullName().toUpperCase());
blackFieldList = fieldBlackFilterMap.get(tableMeta.getFullName().toUpperCase());
}
if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) {
if (tableMetaCache.isOnRDS()) {
// 特殊处理下RDS的场景
@@ -650,7 +664,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
// 不解析最后一列
String rdsRowIdColumnName = "#alibaba_rds_row_id#";
String rdsRowIdColumnName = "__#alibaba_rds_row_id#__";
buffer.nextValue(rdsRowIdColumnName, i, info.type, info.meta, false);
Column.Builder columnBuilder = Column.newBuilder();
columnBuilder.setName(rdsRowIdColumnName);
@@ -664,7 +678,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
columnBuilder.setUpdated(false);
if (needField(fieldList, blackFieldList, columnBuilder.getName())) {
if (isAfter) {
if (isAfter) {
rowDataBuilder.addAfterColumns(columnBuilder.build());
} else {
rowDataBuilder.addBeforeColumns(columnBuilder.build());
@@ -712,7 +726,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
// fixed issue
// https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
boolean isBinary = false;
boolean isSingleBit = false;
if (fieldMeta != null) {
if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
isBinary = true;
@@ -830,7 +843,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
columnBuilder.getIsNull() ? null : columnBuilder.getValue(),
i));
if (needField(fieldList, blackFieldList, columnBuilder.getName())) {
if (isAfter) {
if (isAfter) {
rowDataBuilder.addAfterColumns(columnBuilder.build());
} else {
rowDataBuilder.addBeforeColumns(columnBuilder.build());
@@ -969,16 +982,17 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private boolean isRDSHeartBeat(String schema, String table) {
return "mysql".equalsIgnoreCase(schema) && "ha_health_check".equalsIgnoreCase(table);
}
/**
* 字段过滤判断
*/
private boolean needField(List<String> fieldList, List<String> blackFieldList, String columnName) {
if (fieldList == null || fieldList.isEmpty()) {
return blackFieldList == null || blackFieldList.isEmpty() || !blackFieldList.contains(columnName.toUpperCase());
} else {
return fieldList.contains(columnName.toUpperCase());
}
if (fieldList == null || fieldList.isEmpty()) {
return blackFieldList == null || blackFieldList.isEmpty()
|| !blackFieldList.contains(columnName.toUpperCase());
} else {
return fieldList.contains(columnName.toUpperCase());
}
}
public static TransactionBegin createTransactionBegin(long threadId) {
@@ -1021,31 +1035,30 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
this.nameBlackFilter = nameBlackFilter;
logger.warn("--> init table black filter : " + nameBlackFilter.toString());
}
public void setFieldFilterMap(Map<String, List<String>> fieldFilterMap) {
if (fieldFilterMap != null) {
this.fieldFilterMap = fieldFilterMap;
} else {
this.fieldFilterMap = new HashMap<String, List<String>>();
}
for (Map.Entry<String, List<String>> entry : this.fieldFilterMap.entrySet()) {
logger.warn("--> init field filter : " + entry.getKey() + "->" + entry.getValue());
}
}
if (fieldFilterMap != null) {
this.fieldFilterMap = fieldFilterMap;
} else {
this.fieldFilterMap = new HashMap<String, List<String>>();
}
for (Map.Entry<String, List<String>> entry : this.fieldFilterMap.entrySet()) {
logger.warn("--> init field filter : " + entry.getKey() + "->" + entry.getValue());
}
}
public void setFieldBlackFilterMap(Map<String, List<String>> fieldBlackFilterMap) {
if (fieldBlackFilterMap != null) {
this.fieldBlackFilterMap = fieldBlackFilterMap;
} else {
this.fieldBlackFilterMap = new HashMap<String, List<String>>();
}
for (Map.Entry<String, List<String>> entry : this.fieldBlackFilterMap.entrySet()) {
logger.warn("--> init field black filter : " + entry.getKey() + "->" + entry.getValue());
}
}
if (fieldBlackFilterMap != null) {
this.fieldBlackFilterMap = fieldBlackFilterMap;
} else {
this.fieldBlackFilterMap = new HashMap<String, List<String>>();
}
for (Map.Entry<String, List<String>> entry : this.fieldBlackFilterMap.entrySet()) {
logger.warn("--> init field black filter : " + entry.getKey() + "->" + entry.getValue());
}
}
public void setTableMetaCache(TableMetaCache tableMetaCache) {
this.tableMetaCache = tableMetaCache;

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
View File

@@ -55,7 +55,7 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
rdsLocalBinlogEventParser.setLogPositionManager(this.getLogPositionManager());
rdsLocalBinlogEventParser.setDestination(destination);
rdsLocalBinlogEventParser.setAlarmHandler(this.getAlarmHandler());
rdsLocalBinlogEventParser.setConnectionCharset(this.connectionCharset);
rdsLocalBinlogEventParser.setConnectionCharsetStd(this.connectionCharset);
rdsLocalBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber);
rdsLocalBinlogEventParser.setEnableTsdb(this.enableTsdb);
rdsLocalBinlogEventParser.setEventBlackFilter(this.eventBlackFilter);

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/DescribeBackupPolicyRequest.java
View File

@@ -31,7 +31,7 @@ public class DescribeBackupPolicyRequest extends AbstractRequest<RdsBackupPolicy
JSONObject jsonObj = JSON.parseObject(result);
RdsBackupPolicy policy = new RdsBackupPolicy();
policy.setBackupRetentionPeriod(jsonObj.getString("BackupRetentionPeriod"));
policy.setBackupLog("Enable".equalsIgnoreCase(jsonObj.getString("BackupLog")));
policy.setBackupLog(jsonObj.getString("BackupLog").equalsIgnoreCase("Enable"));
policy.setLogBackupRetentionPeriod(jsonObj.getIntValue("LogBackupRetentionPeriod"));
policy.setPreferredBackupPeriod(jsonObj.getString("PreferredBackupPeriod"));
policy.setPreferredBackupTime(jsonObj.getString("PreferredBackupTime"));

41
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -1,5 +1,27 @@
package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastsql.sql.repository.Schema;
@@ -17,23 +39,6 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
/**
* 基于db远程管理 see internal class: CanalTableMeta , ConsoleTableMetaTSDB
@@ -451,7 +456,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
// 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
continue;
} else if (rollbackPosition.getServerId().equals(snapshotPosition.getServerId())
} else if (rollbackPosition.getServerId() == snapshotPosition.getServerId()
&& snapshotPosition.compareTo(rollbackPosition) > 0) {
continue;
}

35
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -81,8 +81,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter user")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop user")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")) {
String sql = trimSimpleComment(ddl);
repository.console(sql);
repository.console(ddl);
}
} catch (Throwable e) {
logger.warn("parse faield : " + ddl, e);
@@ -98,38 +97,6 @@ public class MemoryTableMeta implements TableMetaTSDB {
return true;
}
/**
* 过滤简单的行与块注释（--或# 开始的行 /*开始与*\/结束的块）
*
* @param sql
* @return
*/
private String trimSimpleComment(String sql) {
String[] sqls = sql.split("\n");
if (sqls.length == 1) return sql;
boolean isCommentBlock = false;
StringBuilder sb = new StringBuilder(sql.length());
for (String psql : sqls) {
if (psql.startsWith("--") || psql.startsWith("#")) {
continue;
} else if (psql.trim().length() == 0) {
sb.append("\n");
}
if (!isCommentBlock && !psql.trim().startsWith("/*") && !psql.trim().endsWith("*/")) {
sb.append(psql).append("\n");
continue;
}
if (!isCommentBlock && psql.trim().startsWith("/*")) {
isCommentBlock = true;
}
if (isCommentBlock && psql.trim().endsWith("*/")) {
isCommentBlock = false;
}
}
return sb.toString();
}
@Override
public TableMeta find(String schema, String table) {
List<String> keys = Arrays.asList(schema, table);