chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

adapter 配置修改动态加载

Browse Source
This commit is contained in:
mcy
2018-11-16 18:07:32 +08:00
parent 9dc4d78b3b
commit d34bee3359
15 changed files with 581 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -1,15 +1,14 @@
package com.alibaba.otter.canal.client.adapter.es;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
@@ -37,12 +36,14 @@ import com.alibaba.otter.canal.client.adapter.support.*;
@SPI("es")
public class ESAdapter implements OuterAdapter {
private Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>(); // 文件名对应配置
private Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = new LinkedHashMap<>(); // schema-table对应配置
private Map<String, ESSyncConfig> esSyncConfig = new ConcurrentHashMap<>(); // 文件名对应配置
private Map<String, Map<String, ESSyncConfig>> dbTableEsSyncConfig = new ConcurrentHashMap<>(); // schema-table对应配置
private TransportClient transportClient;
private TransportClient transportClient;
private ESSyncService esSyncService;
private ESSyncService esSyncService;
private ESConfigMonitor esConfigMonitor;
public TransportClient getTransportClient() {
return transportClient;
@@ -56,7 +57,7 @@ public class ESAdapter implements OuterAdapter {
return esSyncConfig;
}
public Map<String, List<ESSyncConfig>> getDbTableEsSyncConfig() {
public Map<String, Map<String, ESSyncConfig>> getDbTableEsSyncConfig() {
return dbTableEsSyncConfig;
}
@@ -73,7 +74,9 @@ public class ESAdapter implements OuterAdapter {
}
});
for (ESSyncConfig config : esSyncConfig.values()) {
for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
String configName = entry.getKey();
ESSyncConfig config = entry.getValue();
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
config.getEsMapping().setSchemaItem(schemaItem);
@@ -89,9 +92,9 @@ public class ESAdapter implements OuterAdapter {
String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
List<ESSyncConfig> esSyncConfigs = dbTableEsSyncConfig
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new ArrayList<>());
esSyncConfigs.add(config);
Map<String, ESSyncConfig> esSyncConfigMap = dbTableEsSyncConfig
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>());
esSyncConfigMap.put(configName, config);
});
}
@@ -108,6 +111,9 @@ public class ESAdapter implements OuterAdapter {
}
ESTemplate esTemplate = new ESTemplate(transportClient);
esSyncService = new ESSyncService(esTemplate);
esConfigMonitor = new ESConfigMonitor();
esConfigMonitor.init(this);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -117,8 +123,8 @@ public class ESAdapter implements OuterAdapter {
public void sync(Dml dml) {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = dbTableEsSyncConfig.get(database + "-" + table);
esSyncService.sync(esSyncConfigs, dml);
Map<String, ESSyncConfig> configMap = dbTableEsSyncConfig.get(database + "-" + table);
esSyncService.sync(configMap.values(), dml);
}
@Override
@@ -192,6 +198,10 @@ public class ESAdapter implements OuterAdapter {
@Override
public String getDestination(String task) {
if (esConfigMonitor != null) {
esConfigMonitor.destroy();
}
ESSyncConfig config = esSyncConfig.get(task);
if (config != null) {
return config.getDestination();

151
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java Normal file
View File

@@ -0,0 +1,151 @@
package com.alibaba.otter.canal.client.adapter.es.monitor;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;
import org.apache.commons.io.monitor.FileAlterationObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.Util;
public class ESConfigMonitor {
private static final Logger logger = LoggerFactory.getLogger(ESConfigMonitor.class);
private static final String adapterName = "es";
private ESAdapter esAdapter;
private FileAlterationMonitor fileMonitor;
public void init(ESAdapter esAdapter) {
this.esAdapter = esAdapter;
File confDir = Util.getConfDirPath(adapterName);
try {
FileAlterationObserver observer = new FileAlterationObserver(confDir,
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
FileListener listener = new FileListener();
observer.addListener(listener);
fileMonitor = new FileAlterationMonitor(3000, observer);
fileMonitor.start();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
public void destroy() {
try {
fileMonitor.stop();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private class FileListener extends FileAlterationListenerAdaptor {
@Override
public void onFileCreate(File file) {
super.onFileCreate(file);
try {
// 加载新增的配置文件
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class);
config.validate();
addConfigToCache(file, config);
logger.info("Add a new es mapping config: {} to canal adapter", file.getName());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Override
public void onFileChange(File file) {
super.onFileChange(file);
try {
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
// 加载配置文件
String configContent = MappingConfigsLoader
.loadConfig(adapterName + File.separator + file.getName());
ESSyncConfig config = new Yaml().loadAs(configContent, ESSyncConfig.class);
config.validate();
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
deleteConfigFromCache(file);
}
addConfigToCache(file, config);
logger.info("Change a es mapping config: {} of canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Override
public void onFileDelete(File file) {
super.onFileDelete(file);
try {
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
deleteConfigFromCache(file);
logger.info("Delete a es mapping config: {} of canal adapter", file.getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void addConfigToCache(File file, ESSyncConfig config) {
esAdapter.getEsSyncConfig().put(file.getName(), config);
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
config.getEsMapping().setSchemaItem(schemaItem);
DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
if (dataSource == null || dataSource.getUrl() == null) {
throw new RuntimeException("No data source found: " + config.getDataSourceKey());
}
Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
Matcher matcher = pattern.matcher(dataSource.getUrl());
if (!matcher.find()) {
throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
}
String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
Map<String, ESSyncConfig> esSyncConfigMap = esAdapter.getDbTableEsSyncConfig()
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>());
esSyncConfigMap.put(file.getName(), config);
});
}
private void deleteConfigFromCache(File file) {
esAdapter.getEsSyncConfig().remove(file.getName());
for (Map<String, ESSyncConfig> configMap : esAdapter.getDbTableEsSyncConfig().values()) {
if (configMap != null) {
configMap.remove(file.getName());
}
}
}
}
}

3
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.client.adapter.es.service;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -39,7 +40,7 @@ public class ESSyncService {
this.esTemplate = esTemplate;
}
public void sync(List<ESSyncConfig> esSyncConfigs, Dml dml) {
public void sync(Collection<ESSyncConfig> esSyncConfigs, Dml dml) {
long begin = System.currentTimeMillis();
if (esSyncConfigs != null) {
if (logger.isTraceEnabled()) {

12
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java
View File

@@ -50,9 +50,9 @@ public class LabelSyncJoinSub2Test {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;a_", response.getSource().get("_labels"));
@@ -88,9 +88,9 @@ public class LabelSyncJoinSub2Test {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
@@ -120,9 +120,9 @@ public class LabelSyncJoinSub2Test {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b_", response.getSource().get("_labels"));

12
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java
View File

@@ -50,9 +50,9 @@ public class LabelSyncJoinSubTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;a", response.getSource().get("_labels"));
@@ -88,9 +88,9 @@ public class LabelSyncJoinSubTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;aa", response.getSource().get("_labels"));
@@ -120,9 +120,9 @@ public class LabelSyncJoinSubTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b", response.getSource().get("_labels"));

8
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java
View File

@@ -48,9 +48,9 @@ public class RoleSyncJoinOne2Test {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin_", response.getSource().get("_role_name"));
@@ -85,9 +85,9 @@ public class RoleSyncJoinOne2Test {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin3_", response.getSource().get("_role_name"));

18
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java
View File

@@ -49,9 +49,9 @@ public class RoleSyncJoinOneTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin", response.getSource().get("_role_name"));
@@ -86,9 +86,9 @@ public class RoleSyncJoinOneTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin2", response.getSource().get("_role_name"));
@@ -124,9 +124,9 @@ public class RoleSyncJoinOneTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("operator", response.getSource().get("_role_name"));
@@ -151,7 +151,7 @@ public class RoleSyncJoinOneTest {
old2.put("role_id", 2L);
dml2.setOld(oldList2);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml2);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2);
GetResponse response2 = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
@@ -181,9 +181,9 @@ public class RoleSyncJoinOneTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertNull(response.getSource().get("_role_name"));

10
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java
View File

@@ -20,7 +20,7 @@ public class UserSyncJoinOneTest {
@Before
public void init() {
// AdapterConfigs.put("es", "mytest_user_join_one.yml");
// AdapterConfigs.put("es", "mytest_user_join_one.yml");
esAdapter = Common.init();
}
@@ -50,9 +50,9 @@ public class UserSyncJoinOneTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric_", response.getSource().get("_name"));
@@ -86,9 +86,9 @@ public class UserSyncJoinOneTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric2_", response.getSource().get("_name"));

12
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java
View File

@@ -43,9 +43,9 @@ public class UserSyncSingleTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric", response.getSource().get("_name"));
@@ -76,9 +76,9 @@ public class UserSyncSingleTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric2", response.getSource().get("_name"));
@@ -106,9 +106,9 @@ public class UserSyncSingleTest {
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertNull(response.getSource());