添加目标表不区分大小写的支持,并可开关(默认为区分大小写) (#2329)

* 添加目标表不区分大小写的支持,并可开关(默认为区分大小写)

* 添加目标表不区分大小写的支持,并可开关(默认为区分大小写)-format
This commit is contained in:
844392610
2020-02-28 12:22:17 +08:00
committed by GitHub
parent 0d5505aa2c
commit 92f04f86cb
4 changed files with 89 additions and 59 deletions

View File

@@ -101,6 +101,8 @@ public class MappingConfig implements AdapterConfig {
private String targetTable; // 目标表名 private String targetTable; // 目标表名
private Map<String, String> targetColumns; // 目标表字段映射 private Map<String, String> targetColumns; // 目标表字段映射
private boolean caseInsensitive = false; // 目标表不区分大小写,默认是否
private String etlCondition; // etl条件sql private String etlCondition; // etl条件sql
private int readBatch = 5000; private int readBatch = 5000;
@@ -179,6 +181,14 @@ public class MappingConfig implements AdapterConfig {
this.targetColumns = targetColumns; this.targetColumns = targetColumns;
} }
public boolean isCaseInsensitive() {
return caseInsensitive;
}
public void setCaseInsensitive(boolean caseInsensitive) {
this.caseInsensitive = caseInsensitive;
}
public String getEtlCondition() { public String getEtlCondition() {
return etlCondition; return etlCondition;
} }

View File

@@ -91,9 +91,10 @@ public class RdbMirrorDbSyncService {
if (config == null) { if (config == null) {
return false; return false;
} }
// 是否区分大小写
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) { if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData()); int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData());
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
@@ -101,7 +102,7 @@ public class RdbMirrorDbSyncService {
}); });
} else { } else {
int hash = 0; int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
rdbSyncService.getDmlsPartition()[hash].add(syncItem); rdbSyncService.getDmlsPartition()[hash].add(syncItem);

View File

@@ -112,9 +112,8 @@ public class RdbSyncService {
futures.add(executorThreads[i].submit(() -> { futures.add(executorThreads[i].submit(() -> {
try { try {
dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], dmlsPartition[j]
syncItem.config, .forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
syncItem.singleDml));
dmlsPartition[j].clear(); dmlsPartition[j].clear();
batchExecutors[j].commit(); batchExecutors[j].commit();
return true; return true;
@@ -176,8 +175,9 @@ public class RdbSyncService {
} }
for (MappingConfig config : configMap.values()) { for (MappingConfig config : configMap.values()) {
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) { if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
int hash = pkHash(config.getDbMapping(), singleDml.getData()); int hash = pkHash(config.getDbMapping(), singleDml.getData());
SyncItem syncItem = new SyncItem(config, singleDml); SyncItem syncItem = new SyncItem(config, singleDml);
@@ -185,7 +185,7 @@ public class RdbSyncService {
}); });
} else { } else {
int hash = 0; int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
SyncItem syncItem = new SyncItem(config, singleDml); SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem); dmlsPartition[hash].add(syncItem);
@@ -194,7 +194,7 @@ public class RdbSyncService {
} }
return true; return true;
} }
} ); });
} }
/** /**

View File

@@ -5,6 +5,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
import org.springframework.util.LinkedCaseInsensitiveMap;
public class SingleDml { public class SingleDml {
@@ -63,7 +64,7 @@ public class SingleDml {
this.old = old; this.old = old;
} }
public static List<SingleDml> dml2SingleDmls(Dml dml) { public static List<SingleDml> dml2SingleDmls(Dml dml, boolean caseInsensitive) {
List<SingleDml> singleDmls = new ArrayList<>(); List<SingleDml> singleDmls = new ArrayList<>();
if (dml.getData() != null) { if (dml.getData() != null) {
int size = dml.getData().size(); int size = dml.getData().size();
@@ -73,9 +74,17 @@ public class SingleDml {
singleDml.setDatabase(dml.getDatabase()); singleDml.setDatabase(dml.getDatabase());
singleDml.setTable(dml.getTable()); singleDml.setTable(dml.getTable());
singleDml.setType(dml.getType()); singleDml.setType(dml.getType());
singleDml.setData(dml.getData().get(i)); Map<String, Object> data = dml.getData().get(i);
if (caseInsensitive) {
data = toCaseInsensitiveMap(data);
}
singleDml.setData(data);
if (dml.getOld() != null) { if (dml.getOld() != null) {
singleDml.setOld(dml.getOld().get(i)); Map<String, Object> oldData = dml.getOld().get(i);
if (caseInsensitive) {
oldData = toCaseInsensitiveMap(oldData);
}
singleDml.setOld(oldData);
} }
singleDmls.add(singleDml); singleDmls.add(singleDml);
} }
@@ -89,4 +98,14 @@ public class SingleDml {
} }
return singleDmls; return singleDmls;
} }
public static List<SingleDml> dml2SingleDmls(Dml dml) {
return dml2SingleDmls(dml, false);
}
private static <V> LinkedCaseInsensitiveMap<V> toCaseInsensitiveMap(Map<String, V> data) {
LinkedCaseInsensitiveMap map = new LinkedCaseInsensitiveMap();
map.putAll(data);
return map;
}
} }