chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #29 from alibaba/master

Browse Source 
merge
This commit is contained in:
rewerma
2018-12-30 16:50:05 +08:00
committed by GitHub
parent 7ea1468a8d 8d84521767
commit 5e116de264
16 changed files with 301 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -100,7 +100,7 @@ public class ESSyncService {
dml.getDestination(), dml.getDestination(),
config.getEsMapping().get_index()); config.getEsMapping().get_index());
} }
} catch (Exception e) { } catch (Throwable e) {
logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml); logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
throw new RuntimeException(e); throw new RuntimeException(e);
} }

10
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java
View File

@@ -9,7 +9,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -113,6 +118,7 @@ public class HbaseTemplate {
flag = true; flag = true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e);
} }
return flag; return flag;
@@ -145,6 +151,7 @@ public class HbaseTemplate {
flag = true; flag = true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e);
} }
return flag; return flag;
} }
@@ -171,6 +178,7 @@ public class HbaseTemplate {
flag = true; flag = true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
throw new RuntimeException(e);
} }
return flag; return flag;
} }

5
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,7 +2,10 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

28
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -26,7 +26,12 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
import com.alibaba.otter.canal.client.adapter.support.*; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.client.adapter.support.Util;
/** /**
* RDB适配器实现类 * RDB适配器实现类
@@ -73,8 +78,8 @@ public class RdbAdapter implements OuterAdapter {
// 过滤不匹配的key的配置 // 过滤不匹配的key的配置
rdbMappingTmp.forEach((key, mappingConfig) -> { rdbMappingTmp.forEach((key, mappingConfig) -> {
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|| (mappingConfig.getOuterAdapterKey() != null || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
&& mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { .equalsIgnoreCase(configuration.getKey()))) {
rdbMapping.put(key, mappingConfig); rdbMapping.put(key, mappingConfig);
} }
}); });
@@ -87,17 +92,17 @@ public class RdbAdapter implements OuterAdapter {
String configName = entry.getKey(); String configName = entry.getKey();
MappingConfig mappingConfig = entry.getValue(); MappingConfig mappingConfig = entry.getValue();
if (!mappingConfig.getDbMapping().getMirrorDb()) { if (!mappingConfig.getDbMapping().getMirrorDb()) {
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent( String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping() + mappingConfig.getDbMapping().getDatabase() + "."
.getDatabase() + "." + mappingConfig.getDbMapping().getTable(), + mappingConfig.getDbMapping().getTable();
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
k1 -> new ConcurrentHashMap<>()); k1 -> new ConcurrentHashMap<>());
configMap.put(configName, mappingConfig); configMap.put(configName, mappingConfig);
} else { } else {
// mirrorDB // mirrorDB
String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping().getDatabase();
+ mappingConfig.getDbMapping().getDatabase(), mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
MirrorDbConfig.create(configName, mappingConfig));
} }
} }
@@ -110,10 +115,11 @@ public class RdbAdapter implements OuterAdapter {
dataSource.setPassword(properties.get("jdbc.password")); dataSource.setPassword(properties.get("jdbc.password"));
dataSource.setInitialSize(1); dataSource.setInitialSize(1);
dataSource.setMinIdle(1); dataSource.setMinIdle(1);
dataSource.setMaxActive(10); dataSource.setMaxActive(30);
dataSource.setMaxWait(60000); dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000); dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000); dataSource.setMinEvictableIdleTimeMillis(300000);
dataSource.setUseUnfairLock(true);
try { try {
dataSource.init(); dataSource.init();

37
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java
View File

@@ -36,8 +36,7 @@ public class RdbEtlService {
/** /**
* 导入数据 * 导入数据
*/ */
public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, List<String> params) {
List<String> params) {
EtlResult etlResult = new EtlResult(); EtlResult etlResult = new EtlResult();
AtomicLong successCount = new AtomicLong(); AtomicLong successCount = new AtomicLong();
List<String> errMsg = new ArrayList<>(); List<String> errMsg = new ArrayList<>();
@@ -54,8 +53,8 @@ public class RdbEtlService {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
// 拼接sql // 拼接sql
StringBuilder sql = new StringBuilder( StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "."
"SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable()); + dbMapping.getTable());
// 拼接条件 // 拼接条件
appendCondition(params, dbMapping, srcDS, sql); appendCondition(params, dbMapping, srcDS, sql);
@@ -92,8 +91,12 @@ public class RdbEtlService {
} else { } else {
sqlFinal = sql + " LIMIT " + offset + "," + cnt; sqlFinal = sql + " LIMIT " + offset + "," + cnt;
} }
Future<Boolean> future = executor Future<Boolean> future = executor.submit(() -> executeSqlImport(srcDS,
.submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg)); targetDS,
sqlFinal,
dbMapping,
successCount,
errMsg));
futures.add(future); futures.add(future);
} }
@@ -106,10 +109,11 @@ public class RdbEtlService {
executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg); executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg);
} }
logger.info( logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000
dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); + "s!");
etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条"); etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get()
+ " 条");
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
@@ -123,8 +127,8 @@ public class RdbEtlService {
return etlResult; return etlResult;
} }
private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds, private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds, StringBuilder sql)
StringBuilder sql) throws SQLException { throws SQLException {
if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) { if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
AtomicBoolean stExists = new AtomicBoolean(false); AtomicBoolean stExists = new AtomicBoolean(false);
// 验证是否有SYS_TIME字段 // 验证是否有SYS_TIME字段
@@ -188,8 +192,7 @@ public class RdbEtlService {
StringBuilder insertSql = new StringBuilder(); StringBuilder insertSql = new StringBuilder();
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
columnsMap columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
int len = insertSql.length(); int len = insertSql.length();
insertSql.delete(len - 1, len).append(") VALUES ("); insertSql.delete(len - 1, len).append(") VALUES (");
@@ -208,8 +211,8 @@ public class RdbEtlService {
// 删除数据 // 删除数据
Map<String, Object> values = new LinkedHashMap<>(); Map<String, Object> values = new LinkedHashMap<>();
StringBuilder deleteSql = new StringBuilder( StringBuilder deleteSql = new StringBuilder("DELETE FROM " + SyncUtil.getDbTableName(dbMapping)
"DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE "); + " WHERE ");
appendCondition(dbMapping, deleteSql, values, rs); appendCondition(dbMapping, deleteSql, values, rs);
try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) { try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
int k = 1; int k = 1;
@@ -275,8 +278,8 @@ public class RdbEtlService {
/** /**
* 拼接目标表主键where条件 * 拼接目标表主键where条件
*/ */
private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values, private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values, ResultSet rs)
ResultSet rs) throws SQLException { throws SQLException {
// 拼接主键 // 拼接主键
for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) { for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
String targetColumnName = entry.getKey(); String targetColumnName = entry.getKey();

8
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java
View File

@@ -70,8 +70,10 @@ public class RdbMirrorDbSyncService {
} }
} }
if (!dmlList.isEmpty()) { if (!dmlList.isEmpty()) {
rdbSyncService.sync(dmlList, dml -> { rdbSyncService.sync(dmlList,
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase()); dml -> {
MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "."
+ dml.getDatabase());
if (mirrorDbConfig == null) { if (mirrorDbConfig == null) {
return false; return false;
} }
@@ -148,7 +150,7 @@ public class RdbMirrorDbSyncService {
logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase()); logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); throw new RuntimeException(e);
} }
} }
} }

15
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -7,7 +7,11 @@ import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.*; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Function; import java.util.function.Function;
import javax.sql.DataSource; import javax.sql.DataSource;
@@ -74,7 +78,7 @@ public class RdbSyncService {
executorThreads[i] = Executors.newSingleThreadExecutor(); executorThreads[i] = Executors.newSingleThreadExecutor();
} }
} catch (SQLException e) { } catch (SQLException e) {
logger.error(e.getMessage(), e); throw new RuntimeException(e);
} }
} }
@@ -99,12 +103,13 @@ public class RdbSyncService {
int j = i; int j = i;
futures.add(executorThreads[i].submit(() -> { futures.add(executorThreads[i].submit(() -> {
try { try {
dmlsPartition[j] dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); syncItem.config,
syncItem.singleDml));
dmlsPartition[j].clear(); dmlsPartition[j].clear();
batchExecutors[j].commit(); batchExecutors[j].commit();
return true; return true;
} catch (Exception e) { } catch (Throwable e) {
batchExecutors[j].rollback(); batchExecutors[j].rollback();
throw new RuntimeException(e); throw new RuntimeException(e);
} }

78
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
View File

@@ -260,8 +260,86 @@ public final class CharsetConversion {
putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8"); putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8");
putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8"); putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8");
putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8"); putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8");
putEntry(244, "utf8mb4", "utf8mb4_german2_ci", "UTF-8");
putEntry(245, "utf8mb4", "utf8mb4_croatian_ci", "UTF-8");
putEntry(246, "utf8mb4", "utf8mb4_unicode_520_ci", "UTF-8");
putEntry(247, "utf8mb4", "utf8mb4_vietnamese_ci", "UTF-8");
putEntry(248, "gb18030", "gb18030_chinese_ci", "GB18030");
putEntry(249, "gb18030", "gb18030_bin", "GB18030");
putEntry(250, "gb18030", "gb18030_unicode_520_ci", "GB18030");
putEntry(254, "utf8", "utf8_general_cs", "UTF-8"); putEntry(254, "utf8", "utf8_general_cs", "UTF-8");
putEntry(255, "utf8mb4", "utf8mb4_0900_ai_ci", "UTF-8");
putEntry(256, "utf8mb4", "utf8mb4_de_pb_0900_ai_ci", "UTF-8");
putEntry(257, "utf8mb4", "utf8mb4_is_0900_ai_ci", "UTF-8");
putEntry(258, "utf8mb4", "utf8mb4_lv_0900_ai_ci", "UTF-8");
putEntry(259, "utf8mb4", "utf8mb4_ro_0900_ai_ci", "UTF-8");
putEntry(260, "utf8mb4", "utf8mb4_sl_0900_ai_ci", "UTF-8");
putEntry(261, "utf8mb4", "utf8mb4_pl_0900_ai_ci", "UTF-8");
putEntry(262, "utf8mb4", "utf8mb4_et_0900_ai_ci", "UTF-8");
putEntry(263, "utf8mb4", "utf8mb4_es_0900_ai_ci", "UTF-8");
putEntry(264, "utf8mb4", "utf8mb4_sv_0900_ai_ci", "UTF-8");
putEntry(265, "utf8mb4", "utf8mb4_tr_0900_ai_ci", "UTF-8");
putEntry(266, "utf8mb4", "utf8mb4_cs_0900_ai_ci", "UTF-8");
putEntry(267, "utf8mb4", "utf8mb4_da_0900_ai_ci", "UTF-8");
putEntry(268, "utf8mb4", "utf8mb4_lt_0900_ai_ci", "UTF-8");
putEntry(269, "utf8mb4", "utf8mb4_sk_0900_ai_ci", "UTF-8");
putEntry(270, "utf8mb4", "utf8mb4_es_trad_0900_ai_ci", "UTF-8");
putEntry(271, "utf8mb4", "utf8mb4_la_0900_ai_ci", "UTF-8");
putEntry(273, "utf8mb4", "utf8mb4_eo_0900_ai_ci", "UTF-8");
putEntry(274, "utf8mb4", "utf8mb4_hu_0900_ai_ci", "UTF-8");
putEntry(275, "utf8mb4", "utf8mb4_hr_0900_ai_ci", "UTF-8");
putEntry(277, "utf8mb4", "utf8mb4_vi_0900_ai_ci", "UTF-8");
putEntry(278, "utf8mb4", "utf8mb4_0900_as_cs", "UTF-8");
putEntry(279, "utf8mb4", "utf8mb4_de_pb_0900_as_cs", "UTF-8");
putEntry(280, "utf8mb4", "utf8mb4_is_0900_as_cs", "UTF-8");
putEntry(281, "utf8mb4", "utf8mb4_lv_0900_as_cs", "UTF-8");
putEntry(282, "utf8mb4", "utf8mb4_ro_0900_as_cs", "UTF-8");
putEntry(283, "utf8mb4", "utf8mb4_sl_0900_as_cs", "UTF-8");
putEntry(284, "utf8mb4", "utf8mb4_pl_0900_as_cs", "UTF-8");
putEntry(285, "utf8mb4", "utf8mb4_et_0900_as_cs", "UTF-8");
putEntry(286, "utf8mb4", "utf8mb4_es_0900_as_cs", "UTF-8");
putEntry(287, "utf8mb4", "utf8mb4_sv_0900_as_cs", "UTF-8");
putEntry(288, "utf8mb4", "utf8mb4_tr_0900_as_cs", "UTF-8");
putEntry(289, "utf8mb4", "utf8mb4_cs_0900_as_cs", "UTF-8");
putEntry(290, "utf8mb4", "utf8mb4_da_0900_as_cs", "UTF-8");
putEntry(291, "utf8mb4", "utf8mb4_lt_0900_as_cs", "UTF-8");
putEntry(292, "utf8mb4", "utf8mb4_sk_0900_as_cs", "UTF-8");
putEntry(293, "utf8mb4", "utf8mb4_es_trad_0900_as_cs", "UTF-8");
putEntry(294, "utf8mb4", "utf8mb4_la_0900_as_cs", "UTF-8");
putEntry(296, "utf8mb4", "utf8mb4_eo_0900_as_cs", "UTF-8");
putEntry(297, "utf8mb4", "utf8mb4_hu_0900_as_cs", "UTF-8");
putEntry(298, "utf8mb4", "utf8mb4_hr_0900_as_cs", "UTF-8");
putEntry(300, "utf8mb4", "utf8mb4_vi_0900_as_cs", "UTF-8");
putEntry(303, "utf8mb4", "utf8mb4_ja_0900_as_cs", "UTF-8");
putEntry(304, "utf8mb4", "utf8mb4_ja_0900_as_cs_ks", "UTF-8");
putEntry(305, "utf8mb4", "utf8mb4_0900_as_ci", "UTF-8");
putEntry(306, "utf8mb4", "utf8mb4_ru_0900_ai_ci", "UTF-8");
putEntry(307, "utf8mb4", "utf8mb4_ru_0900_as_cs", "UTF-8");
putEntry(326, "utf8mb4", "utf8mb4_test_ci", "UTF-8");
putEntry(327, "utf16", "utf16_test_ci", "UTF-16");
putEntry(328, "utf8mb4", "utf8mb4_test_400_ci", "UTF-8");
putEntry(336, "utf8", "utf8_bengali_standard_ci", "UTF-8");
putEntry(337, "utf8", "utf8_bengali_standard_ci", "UTF-8");
putEntry(352, "utf8", "utf8_phone_ci", "UTF-8");
putEntry(353, "utf8", "utf8_test_ci", "UTF-8");
putEntry(354, "utf8", "utf8_5624_1", "UTF-8");
putEntry(355, "utf8", "utf8_5624_2", "UTF-8");
putEntry(356, "utf8", "utf8_5624_3", "UTF-8");
putEntry(357, "utf8", "utf8_5624_4", "UTF-8");
putEntry(358, "ucs2", "ucs2_test_ci", "UnicodeBig");
putEntry(359, "ucs2", "ucs2_vn_ci", "UnicodeBig");
putEntry(360, "ucs2", "ucs2_5624_1", "UnicodeBig");
putEntry(368, "utf8", "utf8_5624_5", "UTF-8");
putEntry(391, "utf32", "utf32_test_ci", "UTF-32");
putEntry(2047, "utf8", "utf8_maxuserid_ci", "UTF-8");
} }
/** /**

2
deployer/src/main/resources/example/instance.properties
View File

@@ -45,7 +45,7 @@ canal.instance.filter.black.regex=
# mq config # mq config
canal.mq.topic=example canal.mq.topic=example
# 动态topic, 需mq支持动态创建topic # dynamic topic route by table regex
#canal.mq.dynamicTopic=.*,mytest\\..*,mytest2.user #canal.mq.dynamicTopic=.*,mytest\\..*,mytest2.user
canal.mq.partition=0 canal.mq.partition=0
# hash partition config # hash partition config

7
example/src/main/bin/startup.bat
View File

@@ -8,7 +8,7 @@ if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
set conf_dir=%ENV_PATH%\..\conf set conf_dir=%ENV_PATH%\..\conf
set logback_configurationFile=%conf_dir%\logback.xml set logback_configurationFile=%conf_dir%\logback.xml
set client_mode=Simple set client_mode=Simple
if "%1%" != "" set client_mode=%1% if not "%1" == "" set client_mode=%1
set CLASSPATH=%conf_dir% set CLASSPATH=%conf_dir%
set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
@@ -20,7 +20,8 @@ set CANAL_OPTS= -DappName=otter-canal-example -Dlogback.configurationFile="%logb
set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
if "%client_mode%" == "Cluster" if "%client_mode%" == "Cluster" (
java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.ClusterCanalClientTest java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.ClusterCanalClientTest
else ) else (
java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.SimpleCanalClientTest java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.SimpleCanalClientTest
)

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
View File

@@ -20,7 +20,7 @@ public interface ErosaConnection {
/** /**
* 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据 * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据
*/ */
public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException; public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException;
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException; public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;

2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
View File

@@ -75,7 +75,7 @@ public class LocalBinLogConnection implements ErosaConnection {
return running; return running;
} }
public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
} }
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {

10
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
View File

@@ -10,6 +10,7 @@ import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils; import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -118,7 +119,7 @@ public class MysqlConnection implements ErosaConnection {
/** /**
* 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾 * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
*/ */
public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
updateSettings(); updateSettings();
loadBinlogChecksum(); loadBinlogChecksum();
sendBinlogDump(binlogfilename, binlogPosition); sendBinlogDump(binlogfilename, binlogPosition);
@@ -130,6 +131,13 @@ public class MysqlConnection implements ErosaConnection {
decoder.handle(LogEvent.QUERY_EVENT); decoder.handle(LogEvent.QUERY_EVENT);
decoder.handle(LogEvent.XID_EVENT); decoder.handle(LogEvent.XID_EVENT);
LogContext context = new LogContext(); LogContext context = new LogContext();
// 若entry position存在gtid，则使用传入的gtid作为gtidSet 拼接的标准,否则同时开启gtid和tsdb时，会导致丢失gtid
// 而当源端数据库gtid 有purged时会有如下类似报错
// 'errno = 1236, sqlstate = HY000 errmsg = The slave is connecting using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
if (StringUtils.isNotEmpty(gtid)) {
decoder.handle(LogEvent.GTID_LOG_EVENT);
context.setGtidSet(MysqlGTIDSet.parse(gtid));
}
context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum)); context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
while (fetcher.fetch()) { while (fetcher.fetch()) {
accumulateReceivedBytes(fetcher.limit()); accumulateReceivedBytes(fetcher.limit());

7
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
View File

@@ -527,7 +527,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
// 针对开始的第一条为非Begin记录，需要从该binlog扫描 // 针对开始的第一条为非Begin记录，需要从该binlog扫描
final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L); final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
mysqlConnection.reconnect(); mysqlConnection.reconnect();
mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() { mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction<LogEvent>() {
private LogPosition lastPosition; private LogPosition lastPosition;
@@ -740,7 +740,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
try { try {
mysqlConnection.reconnect(); mysqlConnection.reconnect();
// 开始遍历文件 // 开始遍历文件
mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() { mysqlConnection.seek(searchBinlogFile, 4L, endPosition.getGtid(), new SinkFunction<LogEvent>() {
private LogPosition lastPosition; private LogPosition lastPosition;
@@ -754,6 +754,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
event.getLogPos() - event.getEventLen(), event.getLogPos() - event.getEventLen(),
event.getWhen() * 1000, event.getWhen() * 1000,
event.getServerId()); event.getServerId());
entryPosition.setGtid(event.getHeader().getGtidSetStr());
logPosition.setPostion(entryPosition); logPosition.setPostion(entryPosition);
} }
@@ -789,6 +790,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
entryPosition); entryPosition);
} }
logPosition.setPostion(entryPosition); logPosition.setPostion(entryPosition);
entryPosition.setGtid(entry.getHeader().getGtid());
} else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) { } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
// 当前事务开始位点 // 当前事务开始位点
entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId);
@@ -796,6 +798,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
logger.debug("set {} to be pending start position before finding another proper one...", logger.debug("set {} to be pending start position before finding another proper one...",
entryPosition); entryPosition);
} }
entryPosition.setGtid(entry.getHeader().getGtid());
logPosition.setPostion(entryPosition); logPosition.setPostion(entryPosition);
} }