chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

简化adapter配置

Browse Source
This commit is contained in:
mcy
2018-11-12 17:30:40 +08:00
parent 66912d3d05
commit 2e5b7b89af
33 changed files with 260 additions and 536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.client.adapter.rdb;
import java.io.File;
import java.sql.Connection;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
@@ -14,7 +15,7 @@ import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfigLoader;
import com.alibaba.otter.canal.client.adapter.rdb.config.ConfigLoader;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
import com.alibaba.otter.canal.client.adapter.support.*;
@@ -33,12 +34,16 @@ public class RdbAdapter implements OuterAdapter {
@Override
public void init(OuterAdapterConfig configuration) {
SPI spi = this.getClass().getAnnotation(SPI.class);
Map<String, MappingConfig> rdbMappingTmp = MappingConfigLoader.load(spi.value());
System.out.println("xxxxx: " + this.getClass().getClassLoader().getResource("").getPath());
File file = new File(this.getClass().getClassLoader().getResource("").getPath() + "rdb" + File.separator);
System.out.println(file.getAbsolutePath());
Map<String, MappingConfig> rdbMappingTmp = ConfigLoader.load();
// 过滤不匹配的key的配置
rdbMappingTmp.forEach((key, mappingConfig) -> {
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|| mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey())) {
|| (mappingConfig.getOuterAdapterKey() != null
&& mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
rdbMapping.put(key, mappingConfig);
}
});

46
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java Normal file
View File

@@ -0,0 +1,46 @@
package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
/**
* RDB表映射配置加载器
*
* @author rewerma 2018-11-07 下午02:41:34
* @version 1.0.0
*/
public class ConfigLoader {
private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
/**
* 加载HBase表映射配置
*
* @return 配置名/配置文件名--对象
*/
public static Map<String, MappingConfig> load() {
logger.info("## Start loading rdb mapping config ... ");
Map<String, MappingConfig> result = new LinkedHashMap<>();
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("rdb");
configContentMap.forEach((fileName, content) -> {
MappingConfig config = new Yaml().loadAs(content, MappingConfig.class);
try {
config.validate();
} catch (Exception e) {
throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
}
result.put(fileName, config);
});
logger.info("## Rdb mapping config loaded");
return result;
}
}

100
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfigLoader.java
View File

@@ -1,100 +0,0 @@
package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
/**
* RDB表映射配置加载器
*
* @author rewerma 2018-11-07 下午02:41:34
* @version 1.0.0
*/
public class MappingConfigLoader {
private static Logger logger = LoggerFactory.getLogger(MappingConfigLoader.class);
/**
* 加载HBase表映射配置
*
* @return 配置名/配置文件名--对象
*/
public static Map<String, MappingConfig> load(String name) {
logger.info("## Start loading rdb mapping config ... ");
Map<String, MappingConfig> result = new LinkedHashMap<>();
Collection<String> configs = AdapterConfigs.get(name);
if (configs == null) {
return result;
}
for (String c : configs) {
if (c == null) {
continue;
}
c = c.trim();
if (c.equals("") || c.startsWith("#")) {
continue;
}
String configContent = null;
if (c.endsWith(".yml")) {
configContent = readConfigContent(name + "/" + c);
}
MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
try {
config.validate();
} catch (Exception e) {
throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e);
}
result.put(c, config);
}
logger.info("## Rdb mapping config loaded");
return result;
}
public static String readConfigContent(String config) {
InputStream in = null;
try {
// 先取本地文件，再取类路径
File configFile = new File("../conf/" + config);
if (configFile.exists()) {
in = new FileInputStream(configFile);
} else {
in = MappingConfigLoader.class.getClassLoader().getResourceAsStream(config);
}
if (in == null) {
throw new RuntimeException("Rdb mapping config file not found.");
}
byte[] bytes = new byte[in.available()];
in.read(bytes);
return new String(bytes, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new RuntimeException("Read rdb mapping config error. ", e);
} finally {
try {
if (in != null) {
in.close();
}
} catch (IOException e) {
// ignore
}
}
}
}

7
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -148,6 +148,7 @@ public class RdbSyncService {
conn.commit();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
conn.rollback();
} finally {
conn.close();
@@ -230,6 +231,7 @@ public class RdbSyncService {
conn.commit();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
conn.rollback();
} finally {
conn.close();
@@ -283,14 +285,13 @@ public class RdbSyncService {
conn.commit();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
conn.rollback();
} finally {
conn.close();
}
}
/**
* 获取目标字段类型
*
@@ -309,7 +310,7 @@ public class RdbSyncService {
columnType = new LinkedHashMap<>();
final Map<String, Integer> columnTypeTmp = columnType;
String sql = "SELECT * FROM " + dbMapping.getTargetTable() + " WHERE 1=2";
Util.sqlRS(conn, sql, rs -> {
Util.sqlRS(conn, sql, rs -> {
try {
ResultSetMetaData rsd = rs.getMetaData();
int columnCount = rsd.getColumnCount();

6
client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java
View File

@@ -6,23 +6,21 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.client.adapter.rdb.config.ConfigLoader;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfigLoader;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
public class ConfigLoadTest {
@Before
public void before() {
AdapterConfigs.put("oracle", "mytest_user.yml");
// 加载数据源连接池
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
}
@Test
public void testLoad() {
Map<String, MappingConfig> configMap = MappingConfigLoader.load("oracle");
Map<String, MappingConfig> configMap = ConfigLoader.load();
Assert.assertFalse(configMap.isEmpty());
}

2
client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java
View File

@@ -14,7 +14,7 @@ public class Common {
OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
outerAdapterConfig.setName("rdb");
outerAdapterConfig.setKey("oralce1");
//outerAdapterConfig.setKey("oralce1");
Map<String, String> properties = new HashMap<>();
properties.put("jdbc.driveClassName", "oracle.jdbc.OracleDriver");
properties.put("jdbc.url", "jdbc:oracle:thin:@127.0.0.1:49161:XE");

2
client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/OracleSyncTest.java
View File

@@ -6,7 +6,6 @@ import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.Dml;
public class OracleSyncTest {
@@ -15,7 +14,6 @@ public class OracleSyncTest {
@Before
public void init() {
AdapterConfigs.put("rdb", "mytest_user.yml");
rdbAdapter = Common.init();
}

0
client-adapter/rdb/src/test/resources/oracle/mytest_user.yml → client-adapter/rdb/src/test/resources/rdb/mytest_user.yml
View File