chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

modify

Browse Source
This commit is contained in:
mcy
2018-12-12 10:11:06 +08:00
parent 89a4943349
commit 5c4b1f6e63
11 changed files with 158 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -172,7 +172,7 @@ public class RdbAdapter implements OuterAdapter {
public Map<String, Object> count(String task) {
MappingConfig config = rdbMapping.get(task);
MappingConfig.DbMapping dbMapping = config.getDbMapping();
String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.dbTable(dbMapping);
String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping);
Connection conn = null;
Map<String, Object> res = new LinkedHashMap<>();
try {
@@ -198,7 +198,7 @@ public class RdbAdapter implements OuterAdapter {
}
}
}
res.put("targetTable", SyncUtil.dbTable(dbMapping));
res.put("targetTable", SyncUtil.getDbTableName(dbMapping));
return res;
}

21
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.util.LinkedHashMap;
import java.util.Map;
/**
@@ -74,19 +75,19 @@ public class MappingConfig {
public static class DbMapping {
private Boolean mirrorDb = false; // 是否镜像库
private String database; // 数据库名或schema名
private String table; // 表名
private Map<String, String> targetPk; // 目标表主键字段
private Boolean mapAll = false; // 映射所有字段
private String targetDb; // 目标库名
private String targetTable; // 目标表名
private Map<String, String> targetColumns; // 目标表字段映射
private Boolean mirrorDb = false; // 是否镜像库
private String database; // 数据库名或schema名
private String table; // 表名
private Map<String, String> targetPk = new LinkedHashMap<>(); // 目标表主键字段
private Boolean mapAll = false; // 映射所有字段
private String targetDb; // 目标库名
private String targetTable; // 目标表名
private Map<String, String> targetColumns; // 目标表字段映射
private String etlCondition; // etl条件sql
private String etlCondition; // etl条件sql
private int readBatch = 5000;
private int commitBatch = 5000; // etl等批量提交大小
private int commitBatch = 5000; // etl等批量提交大小
public boolean isMirrorDb() {
return mirrorDb == null ? false : mirrorDb;

6
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java
View File

@@ -109,7 +109,7 @@ public class RdbEtlService {
logger.info(
dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
etlResult.setResultMessage("导入目标表 " + SyncUtil.dbTable(dbMapping) + " 数据：" + successCount.get() + " 条");
etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条");
} catch (Exception e) {
logger.error(e.getMessage(), e);
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
@@ -187,7 +187,7 @@ public class RdbEtlService {
// }
StringBuilder insertSql = new StringBuilder();
insertSql.append("INSERT INTO ").append(SyncUtil.dbTable(dbMapping)).append(" (");
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
columnsMap
.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
@@ -209,7 +209,7 @@ public class RdbEtlService {
// 删除数据
Map<String, Object> values = new LinkedHashMap<>();
StringBuilder deleteSql = new StringBuilder(
"DELETE FROM " + SyncUtil.dbTable(dbMapping) + " WHERE ");
"DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE ");
appendCondition(dbMapping, deleteSql, values, rs);
try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
int k = 1;

42
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java
View File

@@ -1,20 +1,14 @@
package com.alibaba.otter.canal.client.adapter.rdb.service;
import java.io.StringWriter;
import java.sql.Connection;
import java.sql.Statement;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
import com.alibaba.fastsql.sql.ast.SQLName;
import com.alibaba.fastsql.sql.ast.SQLStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.fastsql.sql.dialect.mysql.visitor.MySqlOutputVisitor;
import com.alibaba.fastsql.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.fastsql.sql.parser.SQLStatementParser;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,10 +18,13 @@ import com.alibaba.otter.canal.client.adapter.support.Dml;
public class RdbMirrorDbSyncService {
private static final Logger logger = LoggerFactory.getLogger(RdbMirrorDbSyncService.class);
private static final Logger logger = LoggerFactory
.getLogger(RdbMirrorDbSyncService.class);
private Map<String, MappingConfig> mirrorDbConfigCache; // 镜像库配置
private DataSource dataSource;
private Map<String, MappingConfig> mirrorDbConfigCache; // 镜像库配置
private DataSource dataSource;
private static Map<String, MappingConfig> tableDbConfigCache = new ConcurrentHashMap<>();
public RdbMirrorDbSyncService(Map<String, MappingConfig> mirrorDbConfigCache, DataSource dataSource,
Integer threads){
@@ -49,6 +46,29 @@ public class RdbMirrorDbSyncService {
} else {
// DML
// TODO
MappingConfig mappingConfig = tableDbConfigCache
.get(destination + "." + database + "." + dml.getTable());
if (mappingConfig == null) {
// 构造一个配置
mappingConfig = new MappingConfig();
mappingConfig.setDataSourceKey(configMap.getDataSourceKey());
mappingConfig.setDestination(configMap.getDestination());
mappingConfig.setOuterAdapterKey(configMap.getOuterAdapterKey());
mappingConfig.setConcurrent(configMap.getConcurrent());
MappingConfig.DbMapping dbMapping = new MappingConfig.DbMapping();
mappingConfig.setDbMapping(dbMapping);
dbMapping.setDatabase(dml.getDatabase());
dbMapping.setTable(dml.getTable());
dbMapping.setTargetDb(dml.getDatabase());
dbMapping.setTargetTable(dml.getTable());
dbMapping.setMapAll(true);
List<String> pkNames = dml.getPkNames();
Map<String, String> pkMapping = new LinkedHashMap<>();
pkNames.forEach(pkName -> pkMapping.put(pkName, pkName));
dbMapping.setTargetPk(pkMapping);
tableDbConfigCache.put(destination + "." + database + "." + dml.getTable(), mappingConfig);
}
}
}
}

8
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -166,7 +166,7 @@ public class RdbSyncService {
Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
StringBuilder insertSql = new StringBuilder();
insertSql.append("INSERT INTO ").append(SyncUtil.dbTable(dbMapping)).append(" (");
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
int len = insertSql.length();
@@ -230,7 +230,7 @@ public class RdbSyncService {
Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
StringBuilder updateSql = new StringBuilder();
updateSql.append("UPDATE ").append(SyncUtil.dbTable(dbMapping)).append(" SET ");
updateSql.append("UPDATE ").append(SyncUtil.getDbTableName(dbMapping)).append(" SET ");
List<Map<String, ?>> values = new ArrayList<>();
for (String srcColumnName : old.keySet()) {
List<String> targetColumnNames = new ArrayList<>();
@@ -282,7 +282,7 @@ public class RdbSyncService {
Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
StringBuilder sql = new StringBuilder();
sql.append("DELETE FROM ").append(SyncUtil.dbTable(dbMapping)).append(" WHERE ");
sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE ");
List<Map<String, ?>> values = new ArrayList<>();
// 拼接主键
@@ -315,7 +315,7 @@ public class RdbSyncService {
if (columnType == null) {
columnType = new LinkedHashMap<>();
final Map<String, Integer> columnTypeTmp = columnType;
String sql = "SELECT * FROM " + SyncUtil.dbTable(dbMapping) + " WHERE 1=2";
String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2";
Util.sqlRS(conn, sql, rs -> {
try {
ResultSetMetaData rsd = rs.getMetaData();

2
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java
View File

@@ -255,7 +255,7 @@ public class SyncUtil {
}
}
public static String dbTable(MappingConfig.DbMapping dbMapping) {
public static String getDbTableName(MappingConfig.DbMapping dbMapping) {
String result = "";
if (StringUtils.isNotEmpty(dbMapping.getTargetDb())) {
result += dbMapping.getTargetDb() + ".";

1
client-adapter/rdb/src/main/resources/rdb/mytest_user.yml
View File

@@ -6,7 +6,6 @@ dbMapping:
mirrorDb: true
database: mytest
# table: user
targetDb: mytest2
# targetTable: mytest.tb_user
# targetPk:
# id: id