chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #36 from alibaba/master

Browse Source 
merge
This commit is contained in:
rewerma
2019-03-18 09:05:47 +08:00
committed by GitHub
parent 43045bc672 95b26c4d1d
commit 9db2bd291b
5 changed files with 57 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -82,7 +82,7 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
1. canal整体交互协议设计上使用了protobuf3.0，理论上可以支持绝大部分的多语言场景，欢迎大家提交多客户端的PR 1. canal整体交互协议设计上使用了protobuf3.0，理论上可以支持绝大部分的多语言场景，欢迎大家提交多客户端的PR
* canal java客户端: <a href="https://github.com/alibaba/canal/wiki/ClientExample"> https://github.com/alibaba/canal/wiki/ClientExample </a> * canal java客户端: <a href="https://github.com/alibaba/canal/wiki/ClientExample"> https://github.com/alibaba/canal/wiki/ClientExample </a>
* canal c#客户端开源项目地址：<a href="https://github.com/CanalClient/CanalSharp"> https://github.com/CanalSharp/CanalSharp </a> * canal c#客户端开源项目地址：<a href="https://github.com/dotnetcore/CanalSharp"> https://github.com/dotnetcore/CanalSharp </a>
* canal go客户端开源项目地址：<a href="https://github.com/CanalClient/canal-go"> https://github.com/CanalClient/canal-go </a> * canal go客户端开源项目地址：<a href="https://github.com/CanalClient/canal-go"> https://github.com/CanalClient/canal-go </a>
2. canal作为MySQL binlog的增量获取工具，可以将数据投递到MQ系统中，比如Kafka/RocketMQ，可以借助于MQ的多语言能力 2. canal作为MySQL binlog的增量获取工具，可以将数据投递到MQ系统中，比如Kafka/RocketMQ，可以借助于MQ的多语言能力
* 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart) * 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart)
@@ -117,7 +117,7 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
<h3>最新更新</h3> <h3>最新更新</h3>
<ol> <ol>
<li>canal发布重大版本更新1.1.0，具体releaseNode参考：<a href="https://github.com/alibaba/canal/releases/tag/canal-1.1.0">https://github.com/alibaba/canal/releases/tag/canal-1.1.0</a></li> <li>canal发布重大版本更新1.1.0，具体releaseNode参考：<a href="https://github.com/alibaba/canal/releases/tag/canal-1.1.0">https://github.com/alibaba/canal/releases/tag/canal-1.1.0</a></li>
<li>canal c#客户端开源项目地址：<a href="https://github.com/CanalClient/CanalSharp"> https://github.com/CanalClient/CanalSharp </a>，推荐! </li> <li>canal c#客户端开源项目地址：<a href="https://github.com/dotnetcore/CanalSharp"> https://github.com/dotnetcore/CanalSharp </a>，推荐! </li>
<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li> <li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li> <li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java
View File

@@ -76,7 +76,7 @@ public class JdbcTypeUtil {
|| "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType); || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
} }
public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) { public static Object typeConvert(String tableName ,String columnName, String value, int sqlType, String mysqlType) {
if (value == null if (value == null
|| (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) { || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
return null; return null;
@@ -161,7 +161,7 @@ public class JdbcTypeUtil {
} }
return res; return res;
} catch (Exception e) { } catch (Exception e) {
logger.error("table: {} column: {}, failed convert type {} to {}", columnName, value, sqlType); logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
return value; return value;
} }
} }

12
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -77,7 +77,7 @@ public class MessageUtil {
} }
} }
row.put(column.getName(), row.put(column.getName(),
JdbcTypeUtil.typeConvert(column.getName(), JdbcTypeUtil.typeConvert(dml.getTable(),column.getName(),
column.getValue(), column.getValue(),
column.getSqlType(), column.getSqlType(),
column.getMysqlType())); column.getMysqlType()));
@@ -95,7 +95,7 @@ public class MessageUtil {
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) { for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) { if (updateSet.contains(column.getName())) {
rowOld.put(column.getName(), rowOld.put(column.getName(),
JdbcTypeUtil.typeConvert(column.getName(), JdbcTypeUtil.typeConvert(dml.getTable(),column.getName(),
column.getValue(), column.getValue(),
column.getSqlType(), column.getSqlType(),
column.getMysqlType())); column.getMysqlType()));
@@ -153,16 +153,16 @@ public class MessageUtil {
// } // }
List<Map<String, String>> data = flatMessage.getData(); List<Map<String, String>> data = flatMessage.getData();
if (data != null) { if (data != null) {
dml.setData(changeRows(data, flatMessage.getSqlType(), flatMessage.getMysqlType())); dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
} }
List<Map<String, String>> old = flatMessage.getOld(); List<Map<String, String>> old = flatMessage.getOld();
if (old != null) { if (old != null) {
dml.setOld(changeRows(old, flatMessage.getSqlType(), flatMessage.getMysqlType())); dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
} }
return dml; return dml;
} }
private static List<Map<String, Object>> changeRows(List<Map<String, String>> rows, Map<String, Integer> sqlTypes, private static List<Map<String, Object>> changeRows(String table, List<Map<String, String>> rows, Map<String, Integer> sqlTypes,
Map<String, String> mysqlTypes) { Map<String, String> mysqlTypes) {
List<Map<String, Object>> result = new ArrayList<>(); List<Map<String, Object>> result = new ArrayList<>();
for (Map<String, String> row : rows) { for (Map<String, String> row : rows) {
@@ -181,7 +181,7 @@ public class MessageUtil {
continue; continue;
} }
Object finalValue = JdbcTypeUtil.typeConvert(columnName, columnValue, sqlType, mysqlType); Object finalValue = JdbcTypeUtil.typeConvert(table, columnName, columnValue, sqlType, mysqlType);
resultRow.put(columnName, finalValue); resultRow.put(columnName, finalValue);
} }
result.add(resultRow); result.add(resultRow);

7
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -107,8 +107,9 @@ public class RdbSyncService {
int j = i; int j = i;
futures.add(executorThreads[i].submit(() -> { futures.add(executorThreads[i].submit(() -> {
try { try {
dmlsPartition[j] dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); syncItem.config,
syncItem.singleDml));
dmlsPartition[j].clear(); dmlsPartition[j].clear();
batchExecutors[j].commit(); batchExecutors[j].commit();
return true; return true;
@@ -314,7 +315,7 @@ public class RdbSyncService {
for (String srcColumnName : old.keySet()) { for (String srcColumnName : old.keySet()) {
List<String> targetColumnNames = new ArrayList<>(); List<String> targetColumnNames = new ArrayList<>();
columnsMap.forEach((targetColumn, srcColumn) -> { columnsMap.forEach((targetColumn, srcColumn) -> {
if (srcColumnName.toLowerCase().equals(srcColumn.toLowerCase())) { if (srcColumnName.equalsIgnoreCase(srcColumn)) {
targetColumnNames.add(targetColumn); targetColumnNames.add(targetColumn);
} }
}); });

4
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
View File

@@ -106,8 +106,8 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
// stage 3 // stage 3
SequenceBarrier dmlParserSequenceBarrier = disruptorMsgBuffer.newBarrier(simpleParserStage.getSequence()); SequenceBarrier dmlParserSequenceBarrier = disruptorMsgBuffer.newBarrier(simpleParserStage.getSequence());
WorkHandler<MessageEvent>[] workHandlers = new DmlParserStage[parserThreadCount]; WorkHandler<MessageEvent>[] workHandlers = new DmlParserStage[tc];
for (int i = 0; i < parserThreadCount; i++) { for (int i = 0; i < tc; i++) {
workHandlers[i] = new DmlParserStage(); workHandlers[i] = new DmlParserStage();
} }
workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer, workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer,