chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge code and fixed issue #2755 #2756 #2735 #2714 #2711 #2710 #2616 #2602 #2585 #2565 #2935 #2434 #2623 #2418 #2387

Browse Source
This commit is contained in:
agapple
2020-08-22 13:07:24 +08:00
parent 90bf9e6de5
commit b79af46054
66 changed files with 459 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java
View File

@@ -282,7 +282,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
for (PropertyDescriptor descriptor : descriptors) {
String name = descriptor.getName();
if (!"class".equals(name)) {
if (!name.equals("class")) {
RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name);
if (prefixes == null) {
for (String relaxedName : relaxedNames) {

2
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java
View File

@@ -78,7 +78,7 @@ public class JdbcTypeUtil {
public static Object typeConvert(String tableName ,String columnName, String value, int sqlType, String mysqlType) {
if (value == null
|| ("".equals(value) && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
|| (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
return null;
}

1
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -38,6 +38,7 @@ public class MessageUtil {
CanalEntry.EventType eventType = rowChange.getEventType();
final Dml dml = new Dml();
dml.setIsDdl(rowChange.getIsDdl());
dml.setDestination(destination);
dml.setGroupId(groupId);
dml.setDatabase(entry.getHeader().getSchemaName());

6
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java
View File

@@ -95,11 +95,11 @@ public class ESSyncService {
long begin = System.currentTimeMillis();
String type = dml.getType();
if (type != null && "INSERT".equalsIgnoreCase(type)) {
if (type != null && type.equalsIgnoreCase("INSERT")) {
insert(config, dml);
} else if (type != null && "UPDATE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("UPDATE")) {
update(config, dml);
} else if (type != null && "DELETE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("DELETE")) {
delete(config, dml);
} else {
return;

2
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java
View File

@@ -312,7 +312,7 @@ public class MappingConfig implements AdapterConfig {
columnItem.setRowKey(true);
rowKeyColumn = columnItem;
} else {
if (field == null || "".equals(field)) {
if (field == null || field.equals("")) {
columnItem.setFamily(family);
columnItem.setQualifier(columnField.getKey());
} else {

6
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java
View File

@@ -31,11 +31,11 @@ public class HbaseSyncService {
public void sync(MappingConfig config, Dml dml) {
if (config != null) {
String type = dml.getType();
if (type != null && "INSERT".equalsIgnoreCase(type)) {
if (type != null && type.equalsIgnoreCase("INSERT")) {
insert(config, dml);
} else if (type != null && "UPDATE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("UPDATE")) {
update(config, dml);
} else if (type != null && "DELETE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("DELETE")) {
delete(config, dml);
}
if (logger.isDebugEnabled()) {

4
client-adapter/kudu/pom.xml
View File

@@ -34,7 +34,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
<version>5.1.48</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -84,4 +84,4 @@
</plugins>
</build>
</project>
</project>

6
client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduSyncService.java
View File

@@ -47,11 +47,11 @@ public class KuduSyncService {
public void sync(KuduMappingConfig config, Dml dml) {
if (config != null) {
String type = dml.getType();
if (type != null && "INSERT".equalsIgnoreCase(type)) {
if (type != null && type.equalsIgnoreCase("INSERT")) {
insert(config, dml);
} else if (type != null && "UPDATE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("UPDATE")) {
upsert(config, dml);
} else if (type != null && "DELETE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("DELETE")) {
delete(config, dml);
}
if (logger.isDebugEnabled()) {

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java
View File

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.adapter.launcher.config;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
@@ -24,7 +25,7 @@ public class CuratorClient {
@PostConstruct
public void init() {
if (adapterCanalConfig.getZookeeperHosts() != null) {
if (StringUtils.isNotEmpty(adapterCanalConfig.getZookeeperHosts())) {
curator = CuratorFrameworkFactory.builder()
.connectString(adapterCanalConfig.getZookeeperHosts())
.retryPolicy(new ExponentialBackoffRetry(1000, 3))

4
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/rest/CommonRest.java
View File

@@ -181,11 +181,11 @@ public class CommonRest {
*/
@PutMapping("/syncSwitch/{destination}/{status}")
public Result etl(@PathVariable String destination, @PathVariable String status) {
if ("on".equals(status)) {
if (status.equals("on")) {
syncSwitch.on(destination);
logger.info("#Destination: {} sync on", destination);
return Result.createSuccess("实例: " + destination + " 开启同步成功");
} else if ("off".equals(status)) {
} else if (status.equals("off")) {
syncSwitch.off(destination);
logger.info("#Destination: {} sync off", destination);
return Result.createSuccess("实例: " + destination + " 关闭同步成功");

6
client-adapter/pom.xml
View File

@@ -50,7 +50,7 @@
<repositories>
<repository>
<id>central</id>
<url>https://repo1.maven.org/maven2</url>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
@@ -143,7 +143,7 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.196</version>
<version>1.4.192</version>
</dependency>
<!-- 单独引入rocketmq依赖 -->
<dependency>
@@ -172,7 +172,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
<version>5.1.48</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>

103
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -112,12 +112,14 @@ public class RdbSyncService {
futures.add(executorThreads[i].submit(() -> {
try {
dmlsPartition[j]
.forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
syncItem.config,
syncItem.singleDml));
dmlsPartition[j].clear();
batchExecutors[j].commit();
return true;
} catch (Throwable e) {
dmlsPartition[j].clear();
batchExecutors[j].rollback();
throw new RuntimeException(e);
}
@@ -151,50 +153,50 @@ public class RdbSyncService {
sync(dmls, dml -> {
if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
// DDL
columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
return false;
columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
return false;
} else {
// DML
String destination = StringUtils.trimToEmpty(dml.getDestination());
String groupId = StringUtils.trimToEmpty(dml.getGroupId());
String database = dml.getDatabase();
String table = dml.getTable();
Map<String, MappingConfig> configMap;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
} else {
// DML
String destination = StringUtils.trimToEmpty(dml.getDestination());
String groupId = StringUtils.trimToEmpty(dml.getGroupId());
String database = dml.getDatabase();
String table = dml.getTable();
Map<String, MappingConfig> configMap;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
} else {
configMap = mappingConfig.get(destination + "_" + database + "-" + table);
}
if (configMap == null) {
return false;
}
if (configMap.values().isEmpty()) {
return false;
}
for (MappingConfig config : configMap.values()) {
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
int hash = pkHash(config.getDbMapping(), singleDml.getData());
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
} else {
int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
}
}
return true;
configMap = mappingConfig.get(destination + "_" + database + "-" + table);
}
});
if (configMap == null) {
return false;
}
if (configMap.values().isEmpty()) {
return false;
}
for (MappingConfig config : configMap.values()) {
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
int hash = pkHash(config.getDbMapping(), singleDml.getData());
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
} else {
int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> {
SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem);
});
}
}
return true;
}
} );
}
/**
@@ -208,13 +210,13 @@ public class RdbSyncService {
if (config != null) {
try {
String type = dml.getType();
if (type != null && "INSERT".equalsIgnoreCase(type)) {
if (type != null && type.equalsIgnoreCase("INSERT")) {
insert(batchExecutor, config, dml);
} else if (type != null && "UPDATE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("UPDATE")) {
update(batchExecutor, config, dml);
} else if (type != null && "DELETE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("DELETE")) {
delete(batchExecutor, config, dml);
} else if (type != null && "TRUNCATE".equalsIgnoreCase(type)) {
} else if (type != null && type.equalsIgnoreCase("TRUNCATE")) {
truncate(batchExecutor, config);
}
if (logger.isDebugEnabled()) {
@@ -245,7 +247,10 @@ public class RdbSyncService {
StringBuilder insertSql = new StringBuilder();
insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append("`").append(targetColumnName).append("`").append(","));
columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append("`")
.append(targetColumnName)
.append("`")
.append(","));
int len = insertSql.length();
insertSql.delete(len - 1, len).append(") VALUES (");
int mapLen = columnsMap.size();

2
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java
View File

@@ -71,7 +71,7 @@ public class SyncUtil {
if (value instanceof Boolean) {
pstmt.setBoolean(i, (Boolean) value);
} else if (value instanceof String) {
boolean v = !"0".equals(value);
boolean v = !value.equals("0");
pstmt.setBoolean(i, v);
} else if (value instanceof Number) {
boolean v = ((Number) value).intValue() != 0;