chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init rdb sync

Browse Source
This commit is contained in:
mcy
2018-11-07 21:01:50 +08:00
parent 1126d26574
commit 5ae7112192
19 changed files with 1068 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java Normal file
View File

@@ -0,0 +1,86 @@
package com.alibaba.otter.canal.client.adapter.rdb;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfigLoader;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
@SPI("rdb")
public class RdbAdapter implements OuterAdapter {
private static Logger logger = LoggerFactory.getLogger(RdbAdapter.class);
private static volatile Map<String, MappingConfig> rdbMapping = null; // 文件名对应配置
private static volatile Map<String, MappingConfig> mappingConfigCache = null; // 库名-表名对应配置
private DruidDataSource dataSource;
private RdbSyncService rdbSyncService;
@Override
public void init(OuterAdapterConfig configuration) {
if (mappingConfigCache == null) {
synchronized (MappingConfig.class) {
if (mappingConfigCache == null) {
rdbMapping = MappingConfigLoader.load();
mappingConfigCache = new HashMap<>();
for (MappingConfig mappingConfig : rdbMapping.values()) {
mappingConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+ mappingConfig.getDbMapping().getDatabase() + "."
+ mappingConfig.getDbMapping().getTable(),
mappingConfig);
}
}
}
}
Map<String, String> properties = configuration.getProperties();
dataSource = new DruidDataSource();
dataSource.setDriverClassName(properties.get("jdbc.driverClassName"));
dataSource.setUrl(properties.get("jdbc.url"));
dataSource.setUsername(properties.get("jdbc.username"));
dataSource.setPassword(properties.get("jdbc.password"));
dataSource.setInitialSize(1);
dataSource.setMinIdle(1);
dataSource.setMaxActive(2);
dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000);
try {
dataSource.init();
} catch (SQLException e) {
logger.error("ERROR ## failed to initial datasource: " + properties.get("jdbc.url"), e);
}
rdbSyncService = new RdbSyncService(dataSource);
}
@Override
public void sync(Dml dml) {
String destination = StringUtils.trimToEmpty(dml.getDestination());
String database = dml.getDatabase();
String table = dml.getTable();
MappingConfig config = mappingConfigCache.get(destination + "." + database + "." + table);
rdbSyncService.sync(config, dml);
}
@Override
public void destroy() {
if (dataSource != null) {
dataSource.close();
}
}
}

142
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java Normal file
View File

@@ -0,0 +1,142 @@
package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
/**
* RDB表映射配置
*
* @author rewerma 2018-11-07 下午02:41:34
* @version 1.0.0
*/
public class MappingConfig {
private String dataSourceKey; // 数据源key
private String destination; // canal实例或MQ的topic
private DbMapping dbMapping; // db映射配置
public String getDataSourceKey() {
return dataSourceKey;
}
public void setDataSourceKey(String dataSourceKey) {
this.dataSourceKey = dataSourceKey;
}
public DbMapping getDbMapping() {
return dbMapping;
}
public void setDbMapping(DbMapping dbMapping) {
this.dbMapping = dbMapping;
}
public String getDestination() {
return destination;
}
public void setDestination(String destination) {
this.destination = destination;
}
public void validate() {
if (dbMapping.database == null || dbMapping.database.isEmpty()) {
throw new NullPointerException("dbMapping.database");
}
if (dbMapping.table == null || dbMapping.table.isEmpty()) {
throw new NullPointerException("dbMapping.table");
}
if (dbMapping.targetTable == null || dbMapping.targetTable.isEmpty()) {
throw new NullPointerException("dbMapping.targetTable");
}
}
public static class DbMapping {
private String database; // 数据库名或schema名
private String table; // 表面名
private boolean mapAll = false; // 映射所有字段
private String targetTable; // 目标表名
private Map<String, String> targetColums; // 目标表字段映射
private String etlCondition; // etl条件sql
private Set<String> families = new LinkedHashSet<>(); // column family列表
private int readBatch = 5000;
private int commitBatch = 5000; // etl等批量提交大小
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public boolean isMapAll() {
return mapAll;
}
public void setMapAll(boolean mapAll) {
this.mapAll = mapAll;
}
public String getTargetTable() {
return targetTable;
}
public void setTargetTable(String targetTable) {
this.targetTable = targetTable;
}
public Map<String, String> getTargetColums() {
return targetColums;
}
public void setTargetColums(Map<String, String> targetColums) {
this.targetColums = targetColums;
}
public String getEtlCondition() {
return etlCondition;
}
public void setEtlCondition(String etlCondition) {
this.etlCondition = etlCondition;
}
public Set<String> getFamilies() {
return families;
}
public void setFamilies(Set<String> families) {
this.families = families;
}
public int getReadBatch() {
return readBatch;
}
public void setReadBatch(int readBatch) {
this.readBatch = readBatch;
}
public int getCommitBatch() {
return commitBatch;
}
public void setCommitBatch(int commitBatch) {
this.commitBatch = commitBatch;
}
}
}

102
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfigLoader.java Normal file
View File

@@ -0,0 +1,102 @@
package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
/**
* RDB表映射配置加载器
*
* @author rewerma 2018-11-07 下午02:41:34
* @version 1.0.0
*/
public class MappingConfigLoader {
private static Logger logger = LoggerFactory.getLogger(MappingConfigLoader.class);
private static final String BASE_PATH = "rdb";
/**
* 加载HBase表映射配置
*
* @return 配置名/配置文件名--对象
*/
public static Map<String, MappingConfig> load() {
logger.info("## Start loading rdb mapping config ... ");
Map<String, MappingConfig> result = new LinkedHashMap<>();
Collection<String> configs = AdapterConfigs.get("rdb");
if (configs == null) {
return result;
}
for (String c : configs) {
if (c == null) {
continue;
}
c = c.trim();
if (c.equals("") || c.startsWith("#")) {
continue;
}
String configContent = null;
if (c.endsWith(".yml")) {
configContent = readConfigContent(BASE_PATH + "/" + c);
}
MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
try {
config.validate();
} catch (Exception e) {
throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e);
}
result.put(c, config);
}
logger.info("## Rdb mapping config loaded");
return result;
}
public static String readConfigContent(String config) {
InputStream in = null;
try {
// 先取本地文件，再取类路径
File configFile = new File("../config/" + config);
if (configFile.exists()) {
in = new FileInputStream(configFile);
} else {
in = MappingConfigLoader.class.getClassLoader().getResourceAsStream(config);
}
if (in == null) {
throw new RuntimeException("Rdb mapping config file not found.");
}
byte[] bytes = new byte[in.available()];
in.read(bytes);
return new String(bytes, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new RuntimeException("Read rdb mapping config error. ", e);
} finally {
try {
if (in != null) {
in.close();
}
} catch (IOException e) {
// ignore
}
}
}
}

368
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java Normal file
View File

@@ -0,0 +1,368 @@
package com.alibaba.otter.canal.client.adapter.rdb.service;
import java.io.Reader;
import java.io.StringReader;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.sql.*;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import javax.sql.DataSource;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping;
import com.alibaba.otter.canal.client.adapter.support.Dml;
/**
* RDB同步操作业务
*
* @author rewerma 2018-11-7 下午06:45:49
* @version 1.0.0
*/
public class RdbSyncService {
private static Logger logger = LoggerFactory.getLogger(RdbSyncService.class);
private DataSource dataSource;
public RdbSyncService(DataSource dataSource){
this.dataSource = dataSource;
}
public void sync(MappingConfig config, Dml dml) {
try {
if (config != null) {
String type = dml.getType();
if (type != null && type.equalsIgnoreCase("INSERT")) {
insert(config, dml);
} else if (type != null && type.equalsIgnoreCase("UPDATE")) {
// update(config, dml);
} else if (type != null && type.equalsIgnoreCase("DELETE")) {
// delete(config, dml);
}
if (logger.isDebugEnabled()) {
logger.debug("DML: {}", JSON.toJSONString(dml));
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
/**
* 插入操作
*
* @param config 配置项
* @param dml DML数据
*/
private void insert(MappingConfig config, Dml dml) throws SQLException {
List<Map<String, Object>> data = dml.getData();
if (data == null || data.isEmpty()) {
return;
}
DbMapping dbMapping = config.getDbMapping();
int idx = 1;
boolean complete = false;
Connection conn = dataSource.getConnection();
conn.setAutoCommit(false);
StringBuilder insertSql = new StringBuilder();
insertSql.append("INSERT INTO ").append(dbMapping.getTargetTable()).append(" (");
if (!dbMapping.isMapAll()) {
dbMapping.getTargetColums().forEach((targetColumnName, srcColumnName) -> {
insertSql.append(targetColumnName).append(",");
});
int len = insertSql.length();
insertSql.delete(len - 1, len).append(") VALUES (");
int mapLen = dbMapping.getTargetColums().size();
for (int i = 0; i < mapLen; i++) {
insertSql.append("?,");
}
len = insertSql.length();
insertSql.delete(len - 1, len).append(")");
}
PreparedStatement pstmt = conn.prepareStatement(insertSql.toString());
for (Map<String, Object> r : data) {
pstmt.clearParameters();
convertData2DbRow(conn, config, r, pstmt);
pstmt.execute();
if (idx % config.getDbMapping().getCommitBatch() == 0) {
conn.commit();
complete = true;
}
idx++;
}
if (!complete) {
conn.commit();
}
conn.close();
}
private static void sqlRS(Connection conn, String sql, Consumer<ResultSet> consumer) {
Statement stmt = null;
ResultSet rs = null;
try {
stmt = conn.createStatement();
rs = stmt.executeQuery(sql);
consumer.accept(rs);
} catch (SQLException e) {
logger.error(e.getMessage(), e);
} finally {
if (rs != null) {
try {
rs.close();
rs = null;
} catch (SQLException e) {
// ignore
}
}
if (stmt != null) {
try {
stmt.close();
stmt = null;
} catch (SQLException e) {
// ignore
}
}
}
}
private static Map<String, Map<String, Integer>> COLUMNS_TYPE_CACHE = new ConcurrentHashMap<>();
private Map<String, Integer> getTargetColumnType(Connection conn, MappingConfig config) {
DbMapping dbMapping = config.getDbMapping();
String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable();
Map<String, Integer> columnType = COLUMNS_TYPE_CACHE.get(cacheKey);
if (columnType == null) {
synchronized (RdbSyncService.class) {
columnType = COLUMNS_TYPE_CACHE.get(cacheKey);
if (columnType == null) {
columnType = new LinkedHashMap<>();
final Map<String, Integer> columnTypeTmp = columnType;
String sql = "SELECT * FROM " + dbMapping.getTargetTable() + " WHERE 1=2";
sqlRS(conn, sql, rs -> {
try {
ResultSetMetaData rsd = rs.getMetaData();
int columnCount = rsd.getColumnCount();
for (int i = 1; i <= columnCount; i++) {
columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
}
COLUMNS_TYPE_CACHE.put(cacheKey, columnTypeTmp);
} catch (SQLException e) {
logger.error(e.getMessage(), e);
}
});
}
}
}
return columnType;
}
/**
* 新增类型转换
*
* @param config
* @param data
* @param pstmt
* @throws SQLException
*/
private void convertData2DbRow(Connection conn, MappingConfig config, Map<String, Object> data,
PreparedStatement pstmt) throws SQLException {
DbMapping dbMapping = config.getDbMapping();
if (dbMapping.isMapAll()) {
} else {
Map<String, Integer> ctype = getTargetColumnType(conn, config);
int i = 1;
for (Map.Entry<String, String> entry : dbMapping.getTargetColums().entrySet()) {
String targetClassName = entry.getKey();
String srcColumnName = entry.getValue();
if (srcColumnName == null) {
srcColumnName = targetClassName;
}
Integer type = ctype.get(targetClassName.toLowerCase());
Object value = data.get(srcColumnName);
if (value != null) {
if (type == null) {
throw new RuntimeException("No column: " + targetClassName + " found in target db");
}
switch (type) {
case Types.BIT:
case Types.BOOLEAN:
if (value instanceof Boolean) {
pstmt.setBoolean(i, (Boolean) value);
} else if (value instanceof String) {
boolean v = !value.equals("0");
pstmt.setBoolean(i, v);
} else if (value instanceof Number) {
boolean v = ((Number) value).intValue() != 0;
pstmt.setBoolean(i, v);
} else {
pstmt.setNull(i, type);
}
break;
case Types.CHAR:
case Types.NCHAR:
case Types.VARCHAR:
case Types.LONGVARCHAR:
pstmt.setString(i, value.toString());
break;
case Types.TINYINT:
if (value instanceof Number) {
pstmt.setByte(i, ((Number) value).byteValue());
} else if (value instanceof String) {
pstmt.setByte(i, Byte.parseByte((String) value));
} else {
pstmt.setNull(i, type);
}
break;
case Types.SMALLINT:
if (value instanceof Number) {
pstmt.setShort(i, ((Number) value).shortValue());
} else if (value instanceof String) {
pstmt.setShort(i, Short.parseShort((String) value));
} else {
pstmt.setNull(i, type);
}
break;
case Types.INTEGER:
if (value instanceof Number) {
pstmt.setInt(i, ((Number) value).intValue());
} else if (value instanceof String) {
pstmt.setInt(i, Integer.parseInt((String) value));
} else {
pstmt.setNull(i, type);
}
break;
case Types.BIGINT:
if (value instanceof Number) {
pstmt.setLong(i, ((Number) value).longValue());
} else if (value instanceof String) {
pstmt.setLong(i, Long.parseLong((String) value));
} else {
pstmt.setNull(i, type);
}
break;
case Types.DECIMAL:
case Types.NUMERIC:
pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
break;
case Types.REAL:
if (value instanceof Number) {
pstmt.setFloat(i, ((Number) value).floatValue());
} else if (value instanceof String) {
pstmt.setFloat(i, Float.parseFloat((String) value));
} else {
pstmt.setNull(i, type);
}
break;
case Types.FLOAT:
case Types.DOUBLE:
if (value instanceof Number) {
pstmt.setDouble(i, ((Number) value).doubleValue());
} else if (value instanceof String) {
pstmt.setDouble(i, Double.parseDouble((String) value));
} else {
pstmt.setNull(i, type);
}
break;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
if (value instanceof byte[]) {
pstmt.setBytes(i, (byte[]) value);
} else if (value instanceof String) {
pstmt.setBytes(i, ((String) value).getBytes(StandardCharsets.ISO_8859_1));
} else {
pstmt.setNull(i, type);
}
break;
case Types.CLOB:
if (value instanceof byte[]) {
pstmt.setBytes(i, (byte[]) value);
} else if (value instanceof String) {
Reader clobReader = new StringReader((String) value);
pstmt.setCharacterStream(i, clobReader);
} else {
pstmt.setNull(i, type);
}
break;
case Types.DATE:
if (value instanceof Date) {
pstmt.setDate(i, new java.sql.Date(((Date) value).getTime()));
} else if (value instanceof String) {
String v = (String) value;
if (!v.startsWith("0000-00-00")) {
v = v.trim().replace(" ", "T");
DateTime dt = new DateTime(v);
pstmt.setDate(i, new java.sql.Date(dt.toDate().getTime()));
} else {
pstmt.setNull(i, type);
}
} else {
pstmt.setNull(i, type);
}
break;
case Types.TIME:
if (value instanceof Date) {
pstmt.setTime(i, new java.sql.Time(((Date) value).getTime()));
} else if (value instanceof String) {
String v = (String) value;
v = "T" + v;
DateTime dt = new DateTime(v);
pstmt.setTime(i, new Time(dt.toDate().getTime()));
} else {
pstmt.setNull(i, type);
}
break;
case Types.TIMESTAMP:
if (value instanceof Date) {
pstmt.setTimestamp(i, new java.sql.Timestamp(((Date) value).getTime()));
} else if (value instanceof String) {
String v = (String) value;
if (!v.startsWith("0000-00-00")) {
v = v.trim().replace(" ", "T");
DateTime dt = new DateTime(v);
pstmt.setTimestamp(i, new Timestamp(dt.toDate().getTime()));
} else {
pstmt.setNull(i, type);
}
} else {
pstmt.setNull(i, type);
}
break;
default:
pstmt.setObject(i, value, type);
}
} else {
pstmt.setNull(i, type);
}
i++;
}
}
}
}

1
client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter Normal file
View File

@@ -0,0 +1 @@
rdb=com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter

12
client-adapter/rdb/src/main/resources/rdb/mytest_user.yml Normal file
View File

@@ -0,0 +1,12 @@
dataSourceKey: defaultDS
destination: example
dbMapping:
database: mytest
table: user
targetTable: mytest.tb_user
commitBatch: 3000
mapAll: false
columns:
id:
name:
role_id: