chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

mq模式下adapter配置增加groupId属性对应

Browse Source
This commit is contained in:
mcy
2019-02-01 15:18:16 +08:00
parent 597a08ac37
commit 10276e254a
7 changed files with 33 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -93,8 +93,10 @@ public class ESAdapter implements OuterAdapter {
String schema = matcher.group(2); String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> { schemaItem.getAliasTableItems().values().forEach(tableItem -> {
Map<String, ESSyncConfig> esSyncConfigMap = dbTableEsSyncConfig Map<String, ESSyncConfig> esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>()); StringUtils.trimToEmpty(config.getDestination()) + "-" + StringUtils
.trimToEmpty(config.getGroupId()) + "_" + schema + "-" + tableItem.getTableName(),
k -> new ConcurrentHashMap<>());
esSyncConfigMap.put(configName, config); esSyncConfigMap.put(configName, config);
}); });
} }
@@ -132,21 +134,11 @@ public class ESAdapter implements OuterAdapter {
public void sync(Dml dml) { public void sync(Dml dml) {
String database = dml.getDatabase(); String database = dml.getDatabase();
String table = dml.getTable(); String table = dml.getTable();
Map<String, ESSyncConfig> configMap = dbTableEsSyncConfig.get(database + "-" + table); Map<String, ESSyncConfig> configMap = dbTableEsSyncConfig
if (configMap != null) { .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + "_"
List<ESSyncConfig> configs = new ArrayList<>(); + database + "-" + table);
configMap.values().forEach(esConfig -> { if (configMap != null && !configMap.values().isEmpty()) {
if (StringUtils.isNotEmpty(esConfig.getGroupId())) { esSyncService.sync(configMap.values(), dml);
if (esConfig.getGroupId().equals(dml.getGroupId())) {
configs.add(esConfig);
}
} else {
configs.add(esConfig);
}
});
if (!configs.isEmpty()) {
esSyncService.sync(configs, dml);
}
} }
} }

7
client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml
View File

@@ -1,15 +1,14 @@
dataSourceKey: defaultDS dataSourceKey: defaultDS
destination: example destination: example
groupId: g1
esMapping: esMapping:
_index: mytest_user _index: mytest_user
_type: _doc _type: _doc
_id: _id _id: _id
# pk: id # pk: id
sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,
a.c_time as _c_time, c.labels as _labels from user a a.c_time as _c_time from user a
left join role b on b.id=a.role_id left join role b on b.id=a.role_id"
left join (select user_id, group_concat(label order by id desc separator ';') as labels from label
group by user_id) c on c.user_id=a.id"
# objFields: # objFields:
# _labels: array:; # _labels: array:;
etlCondition: "where a.c_time>='{0}'" etlCondition: "where a.c_time>='{0}'"

11
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java
View File

@@ -69,10 +69,11 @@ public class HbaseAdapter implements OuterAdapter {
for (Map.Entry<String, MappingConfig> entry : hbaseMapping.entrySet()) { for (Map.Entry<String, MappingConfig> entry : hbaseMapping.entrySet()) {
String configName = entry.getKey(); String configName = entry.getKey();
MappingConfig mappingConfig = entry.getValue(); MappingConfig mappingConfig = entry.getValue();
String k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." String k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+ mappingConfig.getHbaseMapping().getDatabase() + "." + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+ mappingConfig.getHbaseMapping().getDatabase() + "-"
+ mappingConfig.getHbaseMapping().getTable(); + mappingConfig.getHbaseMapping().getTable();
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(k, k1 -> new HashMap<>()); Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(k, k1 -> new ConcurrentHashMap<>());
configMap.put(configName, mappingConfig); configMap.put(configName, mappingConfig);
} }
@@ -104,9 +105,11 @@ public class HbaseAdapter implements OuterAdapter {
return; return;
} }
String destination = StringUtils.trimToEmpty(dml.getDestination()); String destination = StringUtils.trimToEmpty(dml.getDestination());
String groupId = StringUtils.trimToEmpty(dml.getGroupId());
String database = dml.getDatabase(); String database = dml.getDatabase();
String table = dml.getTable(); String table = dml.getTable();
Map<String, MappingConfig> configMap = mappingConfigCache.get(destination + "." + database + "." + table); Map<String, MappingConfig> configMap = mappingConfigCache
.get(destination + "-" + groupId + "_" + database + "-" + table);
if (configMap != null) { if (configMap != null) {
List<MappingConfig> configs = new ArrayList<>(); List<MappingConfig> configs = new ArrayList<>();
configMap.values().forEach(config -> { configMap.values().forEach(config -> {

1
client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml
View File

@@ -1,5 +1,6 @@
dataSourceKey: defaultDS dataSourceKey: defaultDS
destination: example destination: example
groupId: g1
hbaseMapping: hbaseMapping:
mode: STRING #NATIVE #PHOENIX mode: STRING #NATIVE #PHOENIX
database: mytest # 数据库名 database: mytest # 数据库名

5
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -86,8 +86,9 @@ public class RdbAdapter implements OuterAdapter {
String configName = entry.getKey(); String configName = entry.getKey();
MappingConfig mappingConfig = entry.getValue(); MappingConfig mappingConfig = entry.getValue();
if (!mappingConfig.getDbMapping().getMirrorDb()) { if (!mappingConfig.getDbMapping().getMirrorDb()) {
String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+ mappingConfig.getDbMapping().getDatabase() + "." + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+ mappingConfig.getDbMapping().getDatabase() + "-"
+ mappingConfig.getDbMapping().getTable(); + mappingConfig.getDbMapping().getTable();
Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key, Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
k1 -> new ConcurrentHashMap<>()); k1 -> new ConcurrentHashMap<>());

8
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -147,9 +147,11 @@ public class RdbSyncService {
} else { } else {
// DML // DML
String destination = StringUtils.trimToEmpty(dml.getDestination()); String destination = StringUtils.trimToEmpty(dml.getDestination());
String groupId = StringUtils.trimToEmpty(dml.getGroupId());
String database = dml.getDatabase(); String database = dml.getDatabase();
String table = dml.getTable(); String table = dml.getTable();
Map<String, MappingConfig> configMap = mappingConfig.get(destination + "." + database + "." + table); Map<String, MappingConfig> configMap = mappingConfig
.get(destination + "-" + groupId + "_" + database + "-" + table);
if (configMap == null) { if (configMap == null) {
return false; return false;
@@ -169,7 +171,6 @@ public class RdbSyncService {
return false; return false;
} }
boolean executed = false;
for (MappingConfig config : configs) { for (MappingConfig config : configs) {
if (config.getConcurrent()) { if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
@@ -186,9 +187,8 @@ public class RdbSyncService {
dmlsPartition[hash].add(syncItem); dmlsPartition[hash].add(syncItem);
}); });
} }
executed = true;
} }
return executed; return true;
} }
}); });
} }

8
client-adapter/rdb/src/main/resources/rdb/mytest_user.yml
View File

@@ -1,11 +1,12 @@
dataSourceKey: defaultDS dataSourceKey: defaultDS
destination: example destination: example
outerAdapterKey: oracle1 groupId: g1
outerAdapterKey: mysql1
concurrent: true concurrent: true
dbMapping: dbMapping:
database: mytest database: mytest
table: user table: user
targetTable: mytest.tb_user targetTable: mytest2.user
targetPk: targetPk:
id: id id: id
# mapAll: true # mapAll: true
@@ -17,9 +18,10 @@ dbMapping:
test1: test1:
# Mirror schema synchronize config ## Mirror schema synchronize config
#dataSourceKey: defaultDS #dataSourceKey: defaultDS
#destination: example #destination: example
#groupId: g1
#outerAdapterKey: mysql1 #outerAdapterKey: mysql1
#concurrent: true #concurrent: true
#dbMapping: #dbMapping: