chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

修复测试类报错 (#4516)

Browse Source 
* optimize YAML config loader (#4332)

* fix bug BASE TABLE as table name (#4217)

* fix issues#4328 (#4329)

* docs: add nodejs canal client support (#4260)

Co-authored-by: zhangxunwei <zhangxunwei@dxy.cn>

* fix destination not encoded (#4279)

* 修复Canal指定时间戳启动失效，总是从最新的点位开始同步问题 (#4348)

* 支持用户自定义的CanalAlarmHandler

* RowsLogEvent增加对TableMapLogEvent判空检查，防止NPE异常

* 修复Canal指定时间戳启动失效，总是从最新的点位开始同步问题, issue: #4347

Co-authored-by: 云时 <mingya.wmy@alibaba-inc.com>

* fixed 4334 , support jdk8/jdk11

* fixed issue #4266 , typo

* fixed issue #4243 , support auto register for cluster = null

* fixed issue #4225 , support mysql version >= 8.0.26 heartbeat v2

* fixed issue #4308 , support query_log_event for maraiadb 10.10.1

* ignore compression event

* support jdk11

* support druid 1.2.12

* fixed mariadb 10.x

* sync canal-template.properties

* update fastjson & druid version (#4406)

* 修复升级2.0.4导致兼容的问题

* update druid & fastjson version

* update fastjson version

* update druid & fastjson version

* meta.dat文件数据丢失 (#4397)

* update fastjson & druid version (#4438)

* 局部变量线程安全，优先使用StringBuilder替换StringBuffer (#4472)

Co-authored-by: 夏亮 <xialiang@newerabc.com>

* 1. CanalController stop 需要同时将 embededCanalServer.stop (#4477)

2. ServerRunningMonitor 线程池未正常回收，线程池管理与 start/stop保持一致

* vuln-fix: Use HTTPS instead of HTTP to resolve dependencies (#4437)

This fixes a security vulnerability in this project where the `build.gradle`
files were configuring Gradle to resolve dependencies over HTTP instead of
HTTPS.

Weakness: CWE-829: Inclusion of Functionality from Untrusted Control Sphere
Severity: High
CVSSS: 8.1
Detection: OpenRewrite

Reported-by: Jonathan Leitschuh <Jonathan.Leitschuh@gmail.com>
Signed-off-by: Jonathan Leitschuh <Jonathan.Leitschuh@gmail.com>

Bug-tracker: https://github.com/JLLeitschuh/security-research/issues/9


Co-authored-by: Moderne <team@moderne.io>

Co-authored-by: Moderne <team@moderne.io>

* fix_ETL同步mysql关键字报错 (#4346)

Co-authored-by: foleyang <foleyang@cogobuy.com>

* optimize code

* use compact BigDecimal

* performance optimize , 1. cache string names 2. cache Charset

* performance optimize ,cache integer/long valueof

* 测试类报错

Co-authored-by: gongchangyou <gongchangyou@gmail.com>
Co-authored-by: ChanaLii <316529035@qq.com>
Co-authored-by: zhangxunwei <zhangxunweia@gmail.com>
Co-authored-by: zhangxunwei <zhangxunwei@dxy.cn>
Co-authored-by: tianpeidong <39491687+tianpeidong@users.noreply.github.com>
Co-authored-by: dataccs <dataccs@163.com>
Co-authored-by: 云时 <mingya.wmy@alibaba-inc.com>
Co-authored-by: jianghang.loujh <jianghang.loujh@alibaba-inc.com>
Co-authored-by: 温绍锦 <shaojin.wensj@alibaba-inc.com>
Co-authored-by: noaso <noasoso@gmail.com>
Co-authored-by: HumanPassenger <40585855+HumanPassenger@users.noreply.github.com>
Co-authored-by: 夏亮 <xialiang@newerabc.com>
Co-authored-by: 华仔 <591327356@qq.com>
Co-authored-by: Jonathan Leitschuh <jonathan.leitschuh@gmail.com>
Co-authored-by: Moderne <team@moderne.io>
Co-authored-by: 杰锅不是锅 <1105568074@qq.com>
Co-authored-by: foleyang <foleyang@cogobuy.com>
This commit is contained in:
rewerma
2022-12-06 16:17:01 +08:00
committed by GitHub
parent a96db39420
commit d899981354
89 changed files with 1108 additions and 4473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
View File

@@ -12,6 +12,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.taobao.tddl.dbsync.binlog.event.*;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.slf4j.Logger;
@@ -40,23 +41,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry.Type;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.google.protobuf.ByteString;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
@@ -147,6 +132,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return parseGTIDLogEvent((GtidLogEvent) logEvent);
case LogEvent.HEARTBEAT_LOG_EVENT:
return parseHeartbeatLogEvent((HeartbeatLogEvent) logEvent);
case LogEvent.HEARTBEAT_LOG_EVENT_V2:
return parseHeartbeatV2LogEvent((HeartbeatV2LogEvent) logEvent);
case LogEvent.GTID_EVENT:
case LogEvent.GTID_LIST_EVENT:
return parseMariaGTIDLogEvent(logEvent);
@@ -173,13 +160,22 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
return entryBuilder.build();
}
private Entry parseHeartbeatV2LogEvent(HeartbeatV2LogEvent logEvent) {
Header.Builder headerBuilder = Header.newBuilder();
headerBuilder.setEventType(EventType.MHEARTBEAT);
Entry.Builder entryBuilder = Entry.newBuilder();
entryBuilder.setHeader(headerBuilder.build());
entryBuilder.setEntryType(EntryType.HEARTBEAT);
return entryBuilder.build();
}
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
LogHeader logHeader = logEvent.getHeader();
Pair.Builder builder = Pair.newBuilder();
builder.setKey("gtid");
builder.setValue(logEvent.getGtidStr());
if (logEvent.getLastCommitted() != null) {
if (logEvent.getLastCommitted() != -1) {
builder.setKey("lastCommitted");
builder.setValue(String.valueOf(logEvent.getLastCommitted()));
builder.setKey("sequenceNumber");
@@ -396,7 +392,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
// mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
String queryString = null;
try {
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset);
String tableName = null;
if (useDruidDdlFilter) {
List<DdlResult> results = DruidDdlParser.parse(queryString, null);
@@ -418,7 +414,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
// mariaDb支持，需要设置binlog_annotate_row_events=true，可详细打印原始DML语句
String queryString = null;
try {
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset);
return buildQueryEntry(queryString, event.getHeader());
} catch (UnsupportedEncodingException e) {
throw new CanalParseException(e);
@@ -512,10 +508,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
public void parseTableMapEvent(TableMapLogEvent event) {
try {
String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset.name());
String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset);
event.setDbname(charsetDbName);
String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset.name());
String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset);
event.setTblname(charsetTbName);
} catch (UnsupportedEncodingException e) {
throw new CanalParseException(e);
@@ -554,7 +550,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
rowChangeBuider.setIsDdl(false);
rowChangeBuider.setEventType(eventType);
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
RowsLogBuffer buffer = event.getRowsBuf(charset);
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java
View File

@@ -100,7 +100,7 @@ public class DdlResult {
@Override
public String toString() {
DdlResult ddlResult = this;
StringBuffer sb = new StringBuffer();
StringBuilder sb = new StringBuilder();
do {
sb.append(String.format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];",
ddlResult.schemaName,

1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -209,6 +209,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
for (String schema : schemas) {
// filter views
packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
columnSize = packet.getFieldDescriptors().size();
int tableNameColumnIndex = 0; // default index is 0
List<String> tables = new ArrayList<>();
for (int line = 0; line < packet.getFieldValues().size() / columnSize; line++) {

5
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
View File

@@ -80,7 +80,8 @@ public class MemoryTableMeta implements TableMetaTSDB {
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create user")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter user")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop user")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")) {
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")
&& !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter schema")) {
repository.console(ddl);
}
} catch (Throwable e) {
@@ -145,7 +146,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
public Map<String, String> snapshot() {
Map<String, String> schemaDdls = new HashMap<>();
for (Schema schema : repository.getSchemas()) {
StringBuffer data = new StringBuffer(4 * 1024);
StringBuilder data = new StringBuilder(4 * 1024);
for (String table : schema.showTables()) {
SchemaObject schemaObject = schema.findTable(table);
schemaObject.getStatement().output(data);

2
parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
View File

@@ -182,7 +182,7 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
return null;
}
String json = FileUtils.readFileToString(dataFile, charset.name());
String json = FileUtils.readFileToString(dataFile, charset);
return JsonUtils.unmarshalFromString(json, LogPosition.class);
} catch (IOException e) {
throw new CanalMetaManagerException(e);

2
parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
View File

@@ -302,7 +302,7 @@ public class DirectLogFetcherTest {
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getTable().getDbName(),
event.getTable().getTableName()));
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
RowsLogBuffer buffer = event.getRowsBuf(charset);
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
while (buffer.nextOneRow(columns)) {

2
parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogParsePerformanceTest.java
View File

@@ -134,7 +134,7 @@ public class MysqlBinlogParsePerformanceTest {
public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
try {
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
RowsLogBuffer buffer = event.getRowsBuf(charset);
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
while (buffer.nextOneRow(columns)) {