chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:alibaba/canal

Browse Source
This commit is contained in:
wanshao
2020-02-28 14:24:33 +08:00
parent c21c18458b 5e761e14fa
commit beaca165d0
51 changed files with 1596 additions and 1361 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java
View File

@@ -5,6 +5,10 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import javax.sql.DataSource;
import org.elasticsearch.action.search.SearchResponse;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es6x.etl.ESEtlService; import com.alibaba.otter.canal.client.adapter.es6x.etl.ESEtlService;
@@ -14,9 +18,6 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI; import com.alibaba.otter.canal.client.adapter.support.SPI;
import org.elasticsearch.action.search.SearchResponse;
import javax.sql.DataSource;
/** /**
* ES 6.x 外部适配器 * ES 6.x 外部适配器

22
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java
View File

@@ -10,7 +10,6 @@ import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource; import javax.sql.DataSource;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
@@ -118,8 +117,7 @@ public class ESEtlService extends AbstractEtlService {
if (idVal != null) { if (idVal != null) {
String parentVal = (String) esFieldData.remove("$parent_routing"); String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) { if (mapping.isUpsert()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest( ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_index(),
mapping.get_type(), mapping.get_type(),
idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true); idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
@@ -129,8 +127,9 @@ public class ESEtlService extends AbstractEtlService {
esBulkRequest.add(esUpdateRequest); esBulkRequest.add(esUpdateRequest);
} else { } else {
ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(),
.get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData); mapping.get_type(),
idVal.toString()).setSource(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) { if (StringUtils.isNotEmpty(parentVal)) {
esIndexRequest.setRouting(parentVal); esIndexRequest.setRouting(parentVal);
} }
@@ -139,18 +138,17 @@ public class ESEtlService extends AbstractEtlService {
} else { } else {
idVal = esFieldData.get(mapping.getPk()); idVal = esFieldData.get(mapping.getPk());
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)) mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
.size(10000);
SearchResponse response = esSearchRequest.getResponse(); SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) { for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
.get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData); mapping.get_type(),
hit.getId()).setDoc(esFieldData);
esBulkRequest.add(esUpdateRequest); esBulkRequest.add(esUpdateRequest);
} }
} }
if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 && esBulkRequest.numberOfActions() > 0) {
&& esBulkRequest.numberOfActions() > 0) {
long esBatchBegin = System.currentTimeMillis(); long esBatchBegin = System.currentTimeMillis();
ESBulkResponse rp = esBulkRequest.bulk(); ESBulkResponse rp = esBulkRequest.bulk();
if (rp.hasFailures()) { if (rp.hasFailures()) {
@@ -191,7 +189,7 @@ public class ESEtlService extends AbstractEtlService {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
return count; return count;
}); } );
return true; return true;
} catch (Exception e) { } catch (Exception e) {

8
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java
View File

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es6x.support;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@@ -20,8 +24,8 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest; import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse; import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest; import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest;

17
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java
View File

@@ -5,7 +5,6 @@ import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
@@ -27,7 +26,11 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.*; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.RestHighLevelClientExt;
import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -40,6 +43,8 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
/** /**
* ES 连接器, Transport Rest 两种方式 * ES 连接器, Transport Rest 两种方式
* *
@@ -86,10 +91,9 @@ public class ESConnection {
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) { if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
String[] nameAndPwdArr = nameAndPwd.split(":"); String[] nameAndPwdArr = nameAndPwd.split(":");
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1])); nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback( restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
} }
restHighLevelClient = new RestHighLevelClient(restClientBuilder); restHighLevelClient = new RestHighLevelClient(restClientBuilder);
} }
@@ -418,6 +422,7 @@ public class ESConnection {
} }
} }
@SuppressWarnings("deprecation")
public ESBulkResponse bulk() { public ESBulkResponse bulk() {
if (mode == ESClientMode.TRANSPORT) { if (mode == ESClientMode.TRANSPORT) {
BulkResponse responses = bulkRequestBuilder.execute().actionGet(); BulkResponse responses = bulkRequestBuilder.execute().actionGet();

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync; package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource; import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync; package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource; import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync; package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource; import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync; package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource; import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync; package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource; import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync; package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.GetResponse;
import org.junit.Assert; import org.junit.Assert;

8
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java
View File

@@ -5,20 +5,20 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService; import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter; import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate; import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate;
import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection; import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI; import com.alibaba.otter.canal.client.adapter.support.SPI;
import javax.sql.DataSource;
/** /**
* ES 7.x 外部适配器 * ES 7.x 外部适配器
* *

6
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java
View File

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;

8
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java
View File

@@ -59,6 +59,7 @@ public class ESConnection {
private ESClientMode mode; private ESClientMode mode;
@SuppressWarnings("deprecation")
private TransportClient transportClient; private TransportClient transportClient;
private RestHighLevelClient restHighLevelClient; private RestHighLevelClient restHighLevelClient;
@@ -89,10 +90,9 @@ public class ESConnection {
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) { if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
String[] nameAndPwdArr = nameAndPwd.split(":"); String[] nameAndPwdArr = nameAndPwd.split(":");
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1])); nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback( restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
} }
restHighLevelClient = new RestHighLevelClient(restClientBuilder); restHighLevelClient = new RestHighLevelClient(restClientBuilder);
} }

6
client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java
View File

@@ -24,6 +24,7 @@ import org.junit.Test;
@Ignore @Ignore
public class ES7xTest { public class ES7xTest {
@SuppressWarnings("deprecation")
private TransportClient transportClient; private TransportClient transportClient;
@Before @Before
@@ -65,8 +66,9 @@ public class ES7xTest {
relations.put("parent", "2"); relations.put("parent", "2");
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
bulkRequestBuilder bulkRequestBuilder.add(transportClient.prepareIndex("test", "osm", "2_4")
.add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData)); .setRouting("2")
.setSource(esFieldData));
commit(bulkRequestBuilder); commit(bulkRequestBuilder);
} }

2
client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java
View File

@@ -8,9 +8,9 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection; import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
import org.springframework.util.Assert;
@Ignore @Ignore
public class ESConnectionTest { public class ESConnectionTest {

39
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.client.adapter.es.core; package com.alibaba.otter.canal.client.adapter.es.core;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
@@ -8,7 +7,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
@@ -19,6 +17,7 @@ import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser; import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor; import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor;
import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.EtlResult;
@@ -63,8 +62,8 @@ public abstract class ESAdapter implements OuterAdapter {
// 过滤不匹配的key的配置 // 过滤不匹配的key的配置
esSyncConfigTmp.forEach((key, config) -> { esSyncConfigTmp.forEach((key, config) -> {
if ((config.getOuterAdapterKey() == null && configuration.getKey() == null) if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
|| (config.getOuterAdapterKey() != null || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
&& config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) { .equalsIgnoreCase(configuration.getKey()))) {
esSyncConfig.put(key, config); esSyncConfig.put(key, config);
} }
}); });
@@ -104,12 +103,12 @@ public abstract class ESAdapter implements OuterAdapter {
String table = dml.getTable(); String table = dml.getTable();
Map<String, ESSyncConfig> configMap; Map<String, ESSyncConfig> configMap;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
configMap = dbTableEsSyncConfig configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "-"
.get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + StringUtils.trimToEmpty(dml.getGroupId()) + "_" + database + "-"
+ "_" + database + "-" + table); + table);
} else { } else {
configMap = dbTableEsSyncConfig configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-"
.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table); + table);
} }
if (configMap != null && !configMap.values().isEmpty()) { if (configMap != null && !configMap.values().isEmpty()) {
@@ -155,18 +154,24 @@ public abstract class ESAdapter implements OuterAdapter {
} }
String schema = matcher.group(2); String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> { schemaItem.getAliasTableItems()
.values()
.forEach(tableItem -> {
Map<String, ESSyncConfig> esSyncConfigMap; Map<String, ESSyncConfig> esSyncConfigMap;
if (envProperties != null if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
&& !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
esSyncConfigMap = dbTableEsSyncConfig + "-"
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-" + StringUtils.trimToEmpty(config.getGroupId())
+ StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-" + "_"
+ schema
+ "-"
+ tableItem.getTableName(), + tableItem.getTableName(),
k -> new ConcurrentHashMap<>()); k -> new ConcurrentHashMap<>());
} else { } else {
esSyncConfigMap = dbTableEsSyncConfig esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-" + "_"
+ schema
+ "-"
+ tableItem.getTableName(), + tableItem.getTableName(),
k -> new ConcurrentHashMap<>()); k -> new ConcurrentHashMap<>());
} }

15
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java
View File

@@ -9,8 +9,19 @@ import java.util.stream.Collectors;
import com.alibaba.fastsql.sql.SQLUtils; import com.alibaba.fastsql.sql.SQLUtils;
import com.alibaba.fastsql.sql.ast.SQLExpr; import com.alibaba.fastsql.sql.ast.SQLExpr;
import com.alibaba.fastsql.sql.ast.expr.*; import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.fastsql.sql.ast.statement.*; import com.alibaba.fastsql.sql.ast.expr.SQLCaseExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLCharExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
import com.alibaba.fastsql.sql.ast.statement.SQLJoinTableSource;
import com.alibaba.fastsql.sql.ast.statement.SQLSelectGroupByClause;
import com.alibaba.fastsql.sql.ast.statement.SQLSelectItem;
import com.alibaba.fastsql.sql.ast.statement.SQLSelectStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.fastsql.sql.ast.statement.SQLTableSource;
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.fastsql.sql.parser.ParserException; import com.alibaba.fastsql.sql.parser.ParserException;

27
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java
View File

@@ -1,13 +1,9 @@
package com.alibaba.otter.canal.client.adapter.es.core.monitor; package com.alibaba.otter.canal.client.adapter.es.core.monitor;
import java.io.File; import java.io.File;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor; import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -15,12 +11,9 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.Util; import com.alibaba.otter.canal.client.adapter.support.Util;
@@ -70,8 +63,11 @@ public class ESConfigMonitor {
try { try {
// 加载新增的配置文件 // 加载新增的配置文件
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
ESSyncConfig config = YmlConfigBinder ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties); configContent,
ESSyncConfig.class,
null,
envProperties);
if (config != null) { if (config != null) {
config.validate(); config.validate();
addConfigToCache(file, config); addConfigToCache(file, config);
@@ -89,14 +85,17 @@ public class ESConfigMonitor {
try { try {
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) { if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
// 加载配置文件 // 加载配置文件
String configContent = MappingConfigsLoader String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
.loadConfig(adapterName + File.separator + file.getName()); + file.getName());
if (configContent == null) { if (configContent == null) {
onFileDelete(file); onFileDelete(file);
return; return;
} }
ESSyncConfig config = YmlConfigBinder ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties); configContent,
ESSyncConfig.class,
null,
envProperties);
if (config == null) { if (config == null) {
return; return;
} }

10
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java
View File

@@ -1,22 +1,26 @@
package com.alibaba.otter.canal.client.adapter.es.core.service; package com.alibaba.otter.canal.client.adapter.es.core.service;
import java.util.*; import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource; import javax.sql.DataSource;
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem; import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil; import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;

8
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java
View File

@@ -5,7 +5,13 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.sql.Blob; import java.sql.Blob;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.joda.time.DateTime; import org.joda.time.DateTime;

22
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -5,19 +5,24 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource; import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.support.*;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.support.*; import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
/** /**
@@ -79,7 +84,8 @@ public class HbaseEtlService extends AbstractEtlService {
createTable(); createTable();
// 拼接sql // 拼接sql
String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable() + "`"; String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable()
+ "`";
return super.importData(sql, params); return super.importData(sql, params);
} catch (Exception e) { } catch (Exception e) {
@@ -259,7 +265,7 @@ public class HbaseEtlService extends AbstractEtlService {
// throw new RuntimeException(e); // throw new RuntimeException(e);
} }
return i; return i;
}); } );
return true; return true;
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);

4
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigHolder.java
View File

@@ -1,10 +1,10 @@
package com.alibaba.otter.canal.adapter.launcher.config; package com.alibaba.otter.canal.adapter.launcher.config;
import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
/** /**
* @author L.J.R @ 2019-09-03 * @author L.J.R @ 2019-09-03
* @version 1.1.4 * @version 1.1.4

8
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java
View File

@@ -8,20 +8,20 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch; import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil; import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage; import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer; import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
/** /**
* 适配处理器 * 适配处理器

249
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -1,8 +1,5 @@
package com.alibaba.otter.canal.adapter.launcher.loader; package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -60,123 +57,145 @@ public class CanalAdapterLoader {
} }
canalOuterAdapterGroups.add(canalOuterAdapters); canalOuterAdapterGroups.add(canalOuterAdapters);
AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()+"|"+StringUtils.trimToEmpty(group.getGroupId()), AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()
f-> new AdapterProcessor(canalClientConfig,canalAdapter.getInstance(), + "|"
group.getGroupId(), canalOuterAdapterGroups)); + StringUtils.trimToEmpty(group.getGroupId()),
f -> new AdapterProcessor(canalClientConfig,
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups));
adapterProcessor.start(); adapterProcessor.start();
logger.info("Start adapter for canal-client mq topic: {} succeed", logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
canalAdapter.getInstance() + "-" + group.getGroupId()); + group.getGroupId());
} }
} }
// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { // if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client的适配器 // // 初始化canal-client的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { // for (CanalClientConfig.CanalAdapter canalAdapter :
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); // canalClientConfig.getCanalAdapters()) {
// // List<List<OuterAdapter>> canalOuterAdapterGroups = new
// for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { // CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>(); //
// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { // for (CanalClientConfig.Group connectorGroup :
// loadAdapter(c, canalOutConnectors); // canalAdapter.getGroups()) {
// } // List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
// canalOuterAdapterGroups.add(canalOutConnectors); // for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
// } // loadAdapter(c, canalOutConnectors);
// CanalAdapterWorker worker; // }
// if (StringUtils.isNotEmpty(canalServerHost)) { // canalOuterAdapterGroups.add(canalOutConnectors);
// worker = new CanalAdapterWorker(canalClientConfig, // }
// canalAdapter.getInstance(), // CanalAdapterWorker worker;
// canalServerHost, // if (StringUtils.isNotEmpty(canalServerHost)) {
// zkHosts, // worker = new CanalAdapterWorker(canalClientConfig,
// canalOuterAdapterGroups); // canalAdapter.getInstance(),
// } else if (zkHosts != null) { // canalServerHost,
// worker = new CanalAdapterWorker(canalClientConfig, // zkHosts,
// canalAdapter.getInstance(), // canalOuterAdapterGroups);
// zkHosts, // } else if (zkHosts != null) {
// canalOuterAdapterGroups); // worker = new CanalAdapterWorker(canalClientConfig,
// } else { // canalAdapter.getInstance(),
// throw new RuntimeException("No canal server connector found"); // zkHosts,
// } // canalOuterAdapterGroups);
// canalWorkers.put(canalAdapter.getInstance(), worker); // } else {
// worker.start(); // throw new RuntimeException("No canal server connector found");
// logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); // }
// } // canalWorkers.put(canalAdapter.getInstance(), worker);
// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { // worker.start();
// // 初始化canal-client-kafka的适配器 // logger.info("Start adapter for canal instance: {} succeed",
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { // canalAdapter.getInstance());
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) { // }
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); // } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>(); // // 初始化canal-client-kafka的适配器
// for (OuterAdapterConfig config : group.getOuterAdapters()) { // for (CanalClientConfig.CanalAdapter canalAdapter :
// loadAdapter(config, canalOuterAdapters); // canalClientConfig.getCanalAdapters()) {
// } // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// canalOuterAdapterGroups.add(canalOuterAdapters); // List<List<OuterAdapter>> canalOuterAdapterGroups = new
// // CopyOnWriteArrayList<>();
// CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// canalClientConfig.getMqServers(), // for (OuterAdapterConfig config : group.getOuterAdapters()) {
// canalAdapter.getInstance(), // loadAdapter(config, canalOuterAdapters);
// group.getGroupId(), // }
// canalOuterAdapterGroups, // canalOuterAdapterGroups.add(canalOuterAdapters);
// canalClientConfig.getFlatMessage()); //
// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); // CanalAdapterKafkaWorker canalKafkaWorker = new
// canalKafkaWorker.start(); // CanalAdapterKafkaWorker(canalClientConfig,
// logger.info("Start adapter for canal-client mq topic: {} succeed", // canalClientConfig.getMqServers(),
// canalAdapter.getInstance() + "-" + group.getGroupId()); // canalAdapter.getInstance(),
// } // group.getGroupId(),
// } // canalOuterAdapterGroups,
// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { // canalClientConfig.getFlatMessage());
// // 初始化canal-client-rocketMQ的适配器 // canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" +
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { // group.getGroupId(), canalKafkaWorker);
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) { // canalKafkaWorker.start();
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); // logger.info("Start adapter for canal-client mq topic: {} succeed",
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>(); // canalAdapter.getInstance() + "-" + group.getGroupId());
// for (OuterAdapterConfig config : group.getOuterAdapters()) { // }
// loadAdapter(config, canalOuterAdapters); // }
// } // } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode()))
// canalOuterAdapterGroups.add(canalOuterAdapters); // {
// CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, // // 初始化canal-client-rocketMQ的适配器
// canalClientConfig.getMqServers(), // for (CanalClientConfig.CanalAdapter canalAdapter :
// canalAdapter.getInstance(), // canalClientConfig.getCanalAdapters()) {
// group.getGroupId(), // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// canalOuterAdapterGroups, // List<List<OuterAdapter>> canalOuterAdapterGroups = new
// canalClientConfig.getAccessKey(), // CopyOnWriteArrayList<>();
// canalClientConfig.getSecretKey(), // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// canalClientConfig.getFlatMessage(), // for (OuterAdapterConfig config : group.getOuterAdapters()) {
// canalClientConfig.isEnableMessageTrace(), // loadAdapter(config, canalOuterAdapters);
// canalClientConfig.getCustomizedTraceTopic(), // }
// canalClientConfig.getAccessChannel(), // canalOuterAdapterGroups.add(canalOuterAdapters);
// canalClientConfig.getNamespace()); // CanalAdapterRocketMQWorker rocketMQWorker = new
// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); // CanalAdapterRocketMQWorker(canalClientConfig,
// rocketMQWorker.start(); // canalClientConfig.getMqServers(),
// // canalAdapter.getInstance(),
// logger.info("Start adapter for canal-client mq topic: {} succeed", // group.getGroupId(),
// canalAdapter.getInstance() + "-" + group.getGroupId()); // canalOuterAdapterGroups,
// } // canalClientConfig.getAccessKey(),
// } // canalClientConfig.getSecretKey(),
// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) { // canalClientConfig.getFlatMessage(),
// // 初始化canal-client-rabbitMQ的适配器 // canalClientConfig.isEnableMessageTrace(),
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { // canalClientConfig.getCustomizedTraceTopic(),
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) { // canalClientConfig.getAccessChannel(),
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>(); // canalClientConfig.getNamespace());
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>(); // canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" +
// for (OuterAdapterConfig config : group.getOuterAdapters()) { // group.getGroupId(), rocketMQWorker);
// loadAdapter(config, canalOuterAdapters); // rocketMQWorker.start();
// } //
// canalOuterAdapterGroups.add(canalOuterAdapters); // logger.info("Start adapter for canal-client mq topic: {} succeed",
// CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig, // canalAdapter.getInstance() + "-" + group.getGroupId());
// canalOuterAdapterGroups, // }
// canalAdapter.getInstance(), // }
// group.getGroupId(), // } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode()))
// canalClientConfig.getFlatMessage()); // {
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork); // // 初始化canal-client-rabbitMQ的适配器
// rabbitMQWork.start(); // for (CanalClientConfig.CanalAdapter canalAdapter :
// // canalClientConfig.getCanalAdapters()) {
// logger.info("Start adapter for canal-client mq topic: {} succeed", // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// canalAdapter.getInstance() + "-" + group.getGroupId()); // List<List<OuterAdapter>> canalOuterAdapterGroups = new
// } // CopyOnWriteArrayList<>();
// } // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// // CanalAdapterRabbitMQWork // for (OuterAdapterConfig config : group.getOuterAdapters()) {
// } // loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRabbitMQWorker rabbitMQWork = new
// CanalAdapterRabbitMQWorker(canalClientConfig,
// canalOuterAdapterGroups,
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" +
// group.getGroupId(), rabbitMQWork);
// rabbitMQWork.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// // CanalAdapterRabbitMQWork
// }
} }
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) { private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {

4
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -15,14 +15,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder; import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
/** /**

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java
View File

@@ -5,10 +5,11 @@ import java.io.FileOutputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.utils.CommonUtils;
/** /**
* 远程配置监听器实现 * 远程配置监听器实现
* *

13
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java
View File

@@ -3,9 +3,10 @@ package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
/** /**
* RDB表映射配置 * RDB表映射配置
* *
@@ -101,6 +102,8 @@ public class MappingConfig implements AdapterConfig {
private String targetTable; // 目标表名 private String targetTable; // 目标表名
private Map<String, String> targetColumns; // 目标表字段映射 private Map<String, String> targetColumns; // 目标表字段映射
private boolean caseInsensitive = false; // 目标表不区分大小写，默认是否
private String etlCondition; // etl条件sql private String etlCondition; // etl条件sql
private int readBatch = 5000; private int readBatch = 5000;
@@ -179,6 +182,14 @@ public class MappingConfig implements AdapterConfig {
this.targetColumns = targetColumns; this.targetColumns = targetColumns;
} }
public boolean isCaseInsensitive() {
return caseInsensitive;
}
public void setCaseInsensitive(boolean caseInsensitive) {
this.caseInsensitive = caseInsensitive;
}
public String getEtlCondition() { public String getEtlCondition() {
return etlCondition; return etlCondition;
} }

6
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.rdb.service; package com.alibaba.otter.canal.client.adapter.rdb.service;
import java.sql.*; import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;

7
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java
View File

@@ -91,9 +91,10 @@ public class RdbMirrorDbSyncService {
if (config == null) { if (config == null) {
return false; return false;
} }
// 是否区分大小写
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) { if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData()); int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData());
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
@@ -101,7 +102,7 @@ public class RdbMirrorDbSyncService {
}); });
} else { } else {
int hash = 0; int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml);
rdbSyncService.getDmlsPartition()[hash].add(syncItem); rdbSyncService.getDmlsPartition()[hash].add(syncItem);

12
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -112,9 +112,8 @@ public class RdbSyncService {
futures.add(executorThreads[i].submit(() -> { futures.add(executorThreads[i].submit(() -> {
try { try {
dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], dmlsPartition[j]
syncItem.config, .forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
syncItem.singleDml));
dmlsPartition[j].clear(); dmlsPartition[j].clear();
batchExecutors[j].commit(); batchExecutors[j].commit();
return true; return true;
@@ -176,8 +175,9 @@ public class RdbSyncService {
} }
for (MappingConfig config : configMap.values()) { for (MappingConfig config : configMap.values()) {
boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
if (config.getConcurrent()) { if (config.getConcurrent()) {
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
int hash = pkHash(config.getDbMapping(), singleDml.getData()); int hash = pkHash(config.getDbMapping(), singleDml.getData());
SyncItem syncItem = new SyncItem(config, singleDml); SyncItem syncItem = new SyncItem(config, singleDml);
@@ -185,7 +185,7 @@ public class RdbSyncService {
}); });
} else { } else {
int hash = 0; int hash = 0;
List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml); List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
singleDmls.forEach(singleDml -> { singleDmls.forEach(singleDml -> {
SyncItem syncItem = new SyncItem(config, singleDml); SyncItem syncItem = new SyncItem(config, singleDml);
dmlsPartition[hash].add(syncItem); dmlsPartition[hash].add(syncItem);
@@ -194,7 +194,7 @@ public class RdbSyncService {
} }
return true; return true;
} }
} ); });
} }
/** /**

26
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SingleDml.java
View File

@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.springframework.util.LinkedCaseInsensitiveMap;
import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.Dml;
public class SingleDml { public class SingleDml {
@@ -63,7 +65,7 @@ public class SingleDml {
this.old = old; this.old = old;
} }
public static List<SingleDml> dml2SingleDmls(Dml dml) { public static List<SingleDml> dml2SingleDmls(Dml dml, boolean caseInsensitive) {
List<SingleDml> singleDmls = new ArrayList<>(); List<SingleDml> singleDmls = new ArrayList<>();
if (dml.getData() != null) { if (dml.getData() != null) {
int size = dml.getData().size(); int size = dml.getData().size();
@@ -73,9 +75,17 @@ public class SingleDml {
singleDml.setDatabase(dml.getDatabase()); singleDml.setDatabase(dml.getDatabase());
singleDml.setTable(dml.getTable()); singleDml.setTable(dml.getTable());
singleDml.setType(dml.getType()); singleDml.setType(dml.getType());
singleDml.setData(dml.getData().get(i)); Map<String, Object> data = dml.getData().get(i);
if (caseInsensitive) {
data = toCaseInsensitiveMap(data);
}
singleDml.setData(data);
if (dml.getOld() != null) { if (dml.getOld() != null) {
singleDml.setOld(dml.getOld().get(i)); Map<String, Object> oldData = dml.getOld().get(i);
if (caseInsensitive) {
oldData = toCaseInsensitiveMap(oldData);
}
singleDml.setOld(oldData);
} }
singleDmls.add(singleDml); singleDmls.add(singleDml);
} }
@@ -89,4 +99,14 @@ public class SingleDml {
} }
return singleDmls; return singleDmls;
} }
public static List<SingleDml> dml2SingleDmls(Dml dml) {
return dml2SingleDmls(dml, false);
}
private static <V> LinkedCaseInsensitiveMap<V> toCaseInsensitiveMap(Map<String, V> data) {
LinkedCaseInsensitiveMap map = new LinkedCaseInsensitiveMap();
map.putAll(data);
return map;
}
} }

11
client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/DBTest.java
View File

@@ -11,6 +11,7 @@ import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
@Ignore @Ignore
public class DBTest { public class DBTest {
@@ -39,8 +40,7 @@ public class DBTest {
Connection conn = dataSource.getConnection(); Connection conn = dataSource.getConnection();
conn.setAutoCommit(false); conn.setAutoCommit(false);
PreparedStatement pstmt = conn PreparedStatement pstmt = conn.prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)");
.prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)");
java.util.Date now = new java.util.Date(); java.util.Date now = new java.util.Date();
for (int i = 1; i <= 10000; i++) { for (int i = 1; i <= 10000; i++) {
@@ -67,14 +67,16 @@ public class DBTest {
// ResultSetMetaData rsm = rs.getMetaData(); // ResultSetMetaData rsm = rs.getMetaData();
// int cnt = rsm.getColumnCount(); // int cnt = rsm.getColumnCount();
// for (int i = 1; i <= cnt; i++) { // for (int i = 1; i <= cnt; i++) {
// System.out.println(rsm.getColumnName(i) + " " + rsm.getColumnType(i)); // System.out.println(rsm.getColumnName(i) + " " +
// rsm.getColumnType(i));
// } // }
// rs.close(); // rs.close();
// stmt.close(); // stmt.close();
// PreparedStatement pstmt = conn // PreparedStatement pstmt = conn
// .prepareStatement("insert into tb_user (id,name,role_id,c_time,test1,test2) // .prepareStatement("insert into tb_user
// (id,name,role_id,c_time,test1,test2)
// values (?,?,?,?,?,?)"); // values (?,?,?,?,?,?)");
// pstmt.setBigDecimal(1, new BigDecimal("5")); // pstmt.setBigDecimal(1, new BigDecimal("5"));
// pstmt.setString(2, "test"); // pstmt.setString(2, "test");
@@ -91,6 +93,7 @@ public class DBTest {
dataSource.close(); dataSource.close();
} }
@SuppressWarnings("unused")
private String clob2Str(Clob clob) { private String clob2Str(Clob clob) {
String content = ""; String content = "";
try (Reader is = clob.getCharacterStream(); BufferedReader buff = new BufferedReader(is)) { try (Reader is = clob.getCharacterStream(); BufferedReader buff = new BufferedReader(is)) {

1
client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.client; package com.alibaba.otter.canal.client;
import java.io.IOException;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;

125
common/src/main/java/com/google/common/collect/MigrateMap.java
View File

@@ -1,19 +1,134 @@
package com.google.common.collect; package com.google.common.collect;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
public class MigrateMap { public class MigrateMap {
@SuppressWarnings("deprecation") public static <K, V> ConcurrentMap<K, V> makeComputingMap(CacheBuilder<Object, Object> builder,
public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker,
Function<? super K, ? extends V> computingFunction) { Function<? super K, ? extends V> computingFunction) {
return maker.makeComputingMap(computingFunction); final Function<? super K, ? extends V> function = computingFunction;
LoadingCache<K, V> computingCache = builder.build(new CacheLoader<K, V>() {
@Override
public V load(K key) throws Exception {
return function.apply(key);
}
});
return new MigrateConcurrentMap<K, V>(computingCache);
} }
@SuppressWarnings("deprecation")
public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) { public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
return new MapMaker().makeComputingMap(computingFunction); return makeComputingMap(CacheBuilder.newBuilder(), computingFunction);
}
final static class MigrateConcurrentMap<K, V> implements ConcurrentMap<K, V> {
private final LoadingCache<K, V> computingCache;
private final ConcurrentMap<K, V> cacheView;
MigrateConcurrentMap(LoadingCache<K, V> computingCache){
this.computingCache = computingCache;
this.cacheView = computingCache.asMap();
}
@Override
public int size() {
return cacheView.size();
}
@Override
public boolean isEmpty() {
return cacheView.isEmpty();
}
@Override
public boolean containsKey(Object key) {
return cacheView.containsKey(key);
}
@Override
public boolean containsValue(Object value) {
return cacheView.containsValue(value);
}
@Override
public V get(Object key) {
try {
return computingCache.get((K) key);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
@Override
public V put(K key, V value) {
return cacheView.put(key, value);
}
@Override
public V remove(Object key) {
return cacheView.remove(key);
}
@Override
public void putAll(Map<? extends K, ? extends V> m) {
cacheView.putAll(m);
}
@Override
public void clear() {
cacheView.clear();
}
@Override
public Set<K> keySet() {
return cacheView.keySet();
}
@Override
public Collection<V> values() {
return cacheView.values();
}
@Override
public Set<Entry<K, V>> entrySet() {
return cacheView.entrySet();
}
@Override
public V putIfAbsent(K key, V value) {
return cacheView.putIfAbsent(key, value);
}
@Override
public boolean remove(Object key, Object value) {
return cacheView.remove(key, value);
}
@Override
public boolean replace(K key, V oldValue, V newValue) {
return cacheView.replace(key, oldValue, newValue);
}
@Override
public V replace(K key, V value) {
return cacheView.replace(key, value);
}
@Override
public String toString() {
return cacheView.toString();
}
} }
} }

27
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java
View File

@@ -12,21 +12,18 @@ public class CommonMessage implements Serializable {
private String table; // 表名 private String table; // 表名
private List<String> pkNames; private List<String> pkNames;
private Boolean isDdl; private Boolean isDdl;
private String type; // 类型: // 类型:INSERT/UPDATE/DELETE
// INSERT private String type;
// UPDATE // binlog executeTime, 执行耗时
// DELETE private Long es;
// binlog executeTime // dml build timeStamp, 同步时间
private Long es; // 执行耗时 private Long ts;
// dml build timeStamp // 执行的sql,dml sql为空
private Long ts; // 同步时间 private String sql;
private String sql; // 执行的sql, // 数据列表
// dml private List<Map<String, Object>> data;
// sql为空 // 旧数据列表,用于update,size和data的size一一对应
private List<Map<String, Object>> data; // 数据列表 private List<Map<String, Object>> old;
private List<Map<String, Object>> old; // 旧数据列表,
// 用于update,
// size和data的size一一对应
public String getDatabase() { public String getDatabase() {
return database; return database;

5
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java
View File

@@ -7,13 +7,12 @@ import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler; import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler; import org.apache.oro.text.regex.Perl5Compiler;
import com.google.common.collect.MapMaker; import com.google.common.cache.CacheBuilder;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
public class PatternUtils { public class PatternUtils {
@SuppressWarnings("deprecation") private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(CacheBuilder.newBuilder().softValues(),
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(),
pattern -> { pattern -> {
try { try {
PatternCompiler pc = new Perl5Compiler(); PatternCompiler pc = new Perl5Compiler();

10
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
View File

@@ -20,8 +20,8 @@ import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
@@ -34,8 +34,8 @@ import com.google.protobuf.InvalidProtocolBufferException;
*/ */
public class MQMessageUtils { public class MQMessageUtils {
@SuppressWarnings("deprecation") private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(), .softValues(),
pkHashConfigs -> { pkHashConfigs -> {
List<PartitionData> datas = Lists.newArrayList(); List<PartitionData> datas = Lists.newArrayList();
@@ -73,8 +73,8 @@ public class MQMessageUtils {
return datas; return datas;
}); });
@SuppressWarnings("deprecation") private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(), .softValues(),
new Function<String, List<DynamicTopicData>>() { new Function<String, List<DynamicTopicData>>() {
public List<DynamicTopicData> apply(String pkHashConfigs) { public List<DynamicTopicData> apply(String pkHashConfigs) {

1
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java
View File

@@ -409,7 +409,6 @@ public class ExtensionLoader<T> {
} }
} }
@SuppressWarnings("unused")
private static ClassLoader findClassLoader() { private static ClassLoader findClassLoader() {
return ExtensionLoader.class.getClassLoader(); return ExtensionLoader.class.getClassLoader();
} }

4
connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java
View File

@@ -27,12 +27,8 @@ import com.alibaba.otter.canal.protocol.Message;
@SPI("tcp") @SPI("tcp")
public class CanalTCPConsumer implements CanalMsgConsumer { public class CanalTCPConsumer implements CanalMsgConsumer {
private static final String PREFIX_TCP_CONFIG = "canal.tcp.";
private Long currentBatchId = null; private Long currentBatchId = null;
private CanalConnector canalConnector; private CanalConnector canalConnector;
private int batchSize = 500; private int batchSize = 500;
@Override @Override

2
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
View File

@@ -167,7 +167,7 @@ public class JsonConversion {
int type_byte = buffer.getUint8(); int type_byte = buffer.getUint8();
int position = buffer.position(); int position = buffer.position();
// Then there's the length of the value. // Then there's the length of the value.
int q_max_bytes = (int) Math.min(len, 5); int q_max_bytes = (int) Math.min(len - 1, 5);
long q_tlen = 0; long q_tlen = 0;
long q_str_len = 0; long q_str_len = 0;
long q_n = 0; long q_n = 0;

14
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/JsonConversion_Json_ValueTest.java
View File

@@ -1,22 +1,22 @@
package com.taobao.tddl.dbsync.binlog; package com.taobao.tddl.dbsync.binlog;
import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_enum_type;
import com.taobao.tddl.dbsync.binlog.JsonConversion;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import java.lang.reflect.Method; import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_Value;
import com.taobao.tddl.dbsync.binlog.JsonConversion.Json_enum_type;
public class JsonConversion_Json_ValueTest { public class JsonConversion_Json_ValueTest {
@Rule public final ExpectedException thrown = ExpectedException.none(); @Rule
public final ExpectedException thrown = ExpectedException.none();
@Rule public final Timeout globalTimeout = new Timeout(10000); @SuppressWarnings("deprecation")
@Rule
public final Timeout globalTimeout = new Timeout(10000);
/* testedClasses: JsonConversion_Json_Value */ /* testedClasses: JsonConversion_Json_Value */
// Test written by Diffblue Cover. // Test written by Diffblue Cover.

95
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversionTest.java
View File

@@ -1,21 +1,21 @@
package com.taobao.tddl.dbsync.binlog; package com.taobao.tddl.dbsync.binlog;
import com.taobao.tddl.dbsync.binlog.JsonDiffConversion; import java.lang.reflect.InvocationTargetException;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
public class JsonDiffConversionTest { public class JsonDiffConversionTest {
@Rule public final ExpectedException thrown = ExpectedException.none(); @Rule
public final ExpectedException thrown = ExpectedException.none();
@Rule public final Timeout globalTimeout = new Timeout(10000); @SuppressWarnings("deprecation")
@Rule
public final Timeout globalTimeout = new Timeout(10000);
/* testedClasses: JsonDiffConversion */ /* testedClasses: JsonDiffConversion */
// Test written by Diffblue Cover. // Test written by Diffblue Cover.
@@ -27,11 +27,10 @@ public class JsonDiffConversionTest {
final LogBuffer buffer = new LogBuffer(); final LogBuffer buffer = new LogBuffer();
buffer.position = 28; buffer.position = 28;
buffer.semival = 0; buffer.semival = 0;
final byte[] myByteArray = {(byte)3, (byte)3, (byte)67, (byte)67, (byte)67, (byte)67, final byte[] myByteArray = { (byte) 3, (byte) 3, (byte) 67, (byte) 67, (byte) 67, (byte) 67, (byte) 67,
(byte)67, (byte)66, (byte)67, (byte)66, (byte)67, (byte)66, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 67, (byte) 67, (byte) 66,
(byte)67, (byte)67, (byte)67, (byte)66, (byte)67, (byte)66, (byte) 67, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 67, (byte) 67, (byte) 66, (byte) 67,
(byte)67, (byte)66, (byte)67, (byte)67, (byte)67, (byte)66, (byte) 67, (byte) 67, (byte) 66, (byte) 2, (byte) 66 };
(byte)67, (byte)67, (byte)67, (byte)66, (byte)2, (byte)66};
buffer.buffer = myByteArray; buffer.buffer = myByteArray;
buffer.limit = -1_000_000_065; buffer.limit = -1_000_000_065;
buffer.origin = 1_000_000_096; buffer.origin = 1_000_000_096;
@@ -50,13 +49,10 @@ public class JsonDiffConversionTest {
Assert.assertNotNull(buffer); Assert.assertNotNull(buffer);
Assert.assertEquals(30, buffer.position); Assert.assertEquals(30, buffer.position);
Assert.assertEquals(0, buffer.semival); Assert.assertEquals(0, buffer.semival);
Assert.assertArrayEquals(new byte[] {(byte)3, (byte)3, (byte)67, (byte)67, (byte)67, Assert.assertArrayEquals(new byte[] { (byte) 3, (byte) 3, (byte) 67, (byte) 67, (byte) 67, (byte) 67,
(byte)67, (byte)67, (byte)66, (byte)67, (byte)66, (byte) 67, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 67, (byte) 67,
(byte)67, (byte)66, (byte)67, (byte)67, (byte)67, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 66, (byte) 67, (byte) 67, (byte) 67, (byte) 66,
(byte)66, (byte)67, (byte)66, (byte)67, (byte)66, (byte) 67, (byte) 67, (byte) 67, (byte) 66, (byte) 2, (byte) 66 }, buffer.buffer);
(byte)67, (byte)67, (byte)67, (byte)66, (byte)67,
(byte)67, (byte)67, (byte)66, (byte)2, (byte)66},
buffer.buffer);
Assert.assertEquals(-1_000_000_065, buffer.limit); Assert.assertEquals(-1_000_000_065, buffer.limit);
Assert.assertEquals(1_000_000_096, buffer.origin); Assert.assertEquals(1_000_000_096, buffer.origin);
throw ex; throw ex;
@@ -72,10 +68,10 @@ public class JsonDiffConversionTest {
final LogBuffer buffer = new LogBuffer(); final LogBuffer buffer = new LogBuffer();
buffer.position = 28; buffer.position = 28;
buffer.semival = 0; buffer.semival = 0;
final byte[] myByteArray = { final byte[] myByteArray = { (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2,
(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2,
(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2,
(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)3, (byte)2}; (byte) 3, (byte) 2 };
buffer.buffer = myByteArray; buffer.buffer = myByteArray;
buffer.limit = 31; buffer.limit = 31;
buffer.origin = 0; buffer.origin = 0;
@@ -94,12 +90,10 @@ public class JsonDiffConversionTest {
Assert.assertNotNull(buffer); Assert.assertNotNull(buffer);
Assert.assertEquals(29, buffer.position); Assert.assertEquals(29, buffer.position);
Assert.assertEquals(0, buffer.semival); Assert.assertEquals(0, buffer.semival);
Assert.assertArrayEquals(new byte[] {(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, Assert.assertArrayEquals(new byte[] { (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2,
(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2,
(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2, (byte) 2,
(byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte)2, (byte) 2, (byte) 3, (byte) 2 }, buffer.buffer);
(byte)2, (byte)2, (byte)2, (byte)2, (byte)3, (byte)2},
buffer.buffer);
Assert.assertEquals(31, buffer.limit); Assert.assertEquals(31, buffer.limit);
Assert.assertEquals(0, buffer.origin); Assert.assertEquals(0, buffer.origin);
throw ex; throw ex;
@@ -115,10 +109,10 @@ public class JsonDiffConversionTest {
final LogBuffer buffer = new LogBuffer(); final LogBuffer buffer = new LogBuffer();
buffer.position = 15; buffer.position = 15;
buffer.semival = 0; buffer.semival = 0;
final byte[] myByteArray = { final byte[] myByteArray = { (byte) 1, (byte) 1, (byte) 0, (byte) 0, (byte) 1, (byte) 1, (byte) 1, (byte) 1,
(byte)1, (byte)1, (byte)0, (byte)0, (byte)1, (byte)1, (byte)1, (byte)1, (byte)1, (byte)1, (byte) 1, (byte) 1, (byte) 0, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 0, (byte) 1, (byte) 1,
(byte)0, (byte)1, (byte)1, (byte)1, (byte)1, (byte)0, (byte)1, (byte)1, (byte)0, (byte)0, (byte) 0, (byte) 0, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 0,
(byte)1, (byte)1, (byte)1, (byte)1, (byte)1, (byte)1, (byte)1, (byte)0, (byte)1, (byte)1}; (byte) 1, (byte) 1 };
buffer.buffer = myByteArray; buffer.buffer = myByteArray;
buffer.limit = -1_215_751_986; buffer.limit = -1_215_751_986;
buffer.origin = 1_215_752_002; buffer.origin = 1_215_752_002;
@@ -137,12 +131,10 @@ public class JsonDiffConversionTest {
Assert.assertNotNull(buffer); Assert.assertNotNull(buffer);
Assert.assertEquals(16, buffer.position); Assert.assertEquals(16, buffer.position);
Assert.assertEquals(0, buffer.semival); Assert.assertEquals(0, buffer.semival);
Assert.assertArrayEquals(new byte[] {(byte)1, (byte)1, (byte)0, (byte)0, (byte)1, (byte)1, Assert.assertArrayEquals(new byte[] { (byte) 1, (byte) 1, (byte) 0, (byte) 0, (byte) 1, (byte) 1, (byte) 1,
(byte)1, (byte)1, (byte)1, (byte)1, (byte)0, (byte)1, (byte) 1, (byte) 1, (byte) 1, (byte) 0, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 0, (byte) 1,
(byte)1, (byte)1, (byte)1, (byte)0, (byte)1, (byte)1, (byte) 1, (byte) 0, (byte) 0, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 1, (byte) 1,
(byte)0, (byte)0, (byte)1, (byte)1, (byte)1, (byte)1, (byte) 0, (byte) 1, (byte) 1 }, buffer.buffer);
(byte)1, (byte)1, (byte)1, (byte)0, (byte)1, (byte)1},
buffer.buffer);
Assert.assertEquals(-1_215_751_986, buffer.limit); Assert.assertEquals(-1_215_751_986, buffer.limit);
Assert.assertEquals(1_215_752_002, buffer.origin); Assert.assertEquals(1_215_752_002, buffer.origin);
throw ex; throw ex;
@@ -158,10 +150,10 @@ public class JsonDiffConversionTest {
final LogBuffer buffer = new LogBuffer(); final LogBuffer buffer = new LogBuffer();
buffer.position = 27; buffer.position = 27;
buffer.semival = 0; buffer.semival = 0;
final byte[] myByteArray = { final byte[] myByteArray = { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)1, (byte)0}; (byte) 1, (byte) 0 };
buffer.buffer = myByteArray; buffer.buffer = myByteArray;
buffer.limit = 31; buffer.limit = 31;
buffer.origin = -1; buffer.origin = -1;
@@ -180,12 +172,10 @@ public class JsonDiffConversionTest {
Assert.assertNotNull(buffer); Assert.assertNotNull(buffer);
Assert.assertEquals(29, buffer.position); Assert.assertEquals(29, buffer.position);
Assert.assertEquals(0, buffer.semival); Assert.assertEquals(0, buffer.semival);
Assert.assertArrayEquals(new byte[] {(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, Assert.assertArrayEquals(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte) 0, (byte) 1, (byte) 0 }, buffer.buffer);
(byte)0, (byte)0, (byte)0, (byte)0, (byte)1, (byte)0},
buffer.buffer);
Assert.assertEquals(31, buffer.limit); Assert.assertEquals(31, buffer.limit);
Assert.assertEquals(-1, buffer.origin); Assert.assertEquals(-1, buffer.origin);
throw ex; throw ex;
@@ -204,8 +194,11 @@ public class JsonDiffConversionTest {
final String charsetName = "1a 2b 3c"; final String charsetName = "1a 2b 3c";
// Act // Act
final StringBuilder actual = final StringBuilder actual = JsonDiffConversion.print_json_diff(buffer,
JsonDiffConversion.print_json_diff(buffer, len, columnName, columnIndex, charsetName); len,
columnName,
columnIndex,
charsetName);
// Assert result // Assert result
Assert.assertNotNull(actual); Assert.assertNotNull(actual);

8
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogEventTest.java
View File

@@ -1,6 +1,5 @@
package com.taobao.tddl.dbsync.binlog; package com.taobao.tddl.dbsync.binlog;
import com.taobao.tddl.dbsync.binlog.LogEvent;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
@@ -9,9 +8,12 @@ import org.junit.rules.Timeout;
public class LogEventTest { public class LogEventTest {
@Rule public final ExpectedException thrown = ExpectedException.none(); @Rule
public final ExpectedException thrown = ExpectedException.none();
@Rule public final Timeout globalTimeout = new Timeout(10000); @SuppressWarnings("deprecation")
@Rule
public final Timeout globalTimeout = new Timeout(10000);
/* testedClasses: LogEvent */ /* testedClasses: LogEvent */
// Test written by Diffblue Cover. // Test written by Diffblue Cover.

12
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/event/LogHeaderTest.java
View File

@@ -1,19 +1,21 @@
package com.taobao.tddl.dbsync.binlog.event; package com.taobao.tddl.dbsync.binlog.event;
import com.taobao.tddl.dbsync.binlog.event.LogHeader; import java.util.HashMap;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import java.util.HashMap;
public class LogHeaderTest { public class LogHeaderTest {
@Rule public final ExpectedException thrown = ExpectedException.none(); @Rule
public final ExpectedException thrown = ExpectedException.none();
@Rule public final Timeout globalTimeout = new Timeout(10000); @SuppressWarnings("deprecation")
@Rule
public final Timeout globalTimeout = new Timeout(10000);
/* testedClasses: LogHeader */ /* testedClasses: LogHeader */
// Test written by Diffblue Cover. // Test written by Diffblue Cover.

29
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/RegisterSlaveCommandPacketTest.java
View File

@@ -1,21 +1,22 @@
package com.alibaba.otter.canal.parse.driver.mysql.packets.client; package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket; import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import java.io.IOException;
import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
public class RegisterSlaveCommandPacketTest { public class RegisterSlaveCommandPacketTest {
@Rule public final ExpectedException thrown = ExpectedException.none(); @Rule
public final ExpectedException thrown = ExpectedException.none();
@Rule public final Timeout globalTimeout = new Timeout(10000); @SuppressWarnings("deprecation")
@Rule
public final Timeout globalTimeout = new Timeout(10000);
/* testedClasses: RegisterSlaveCommandPacket */ /* testedClasses: RegisterSlaveCommandPacket */
// Test written by Diffblue Cover. // Test written by Diffblue Cover.
@@ -29,18 +30,16 @@ public class RegisterSlaveCommandPacketTest {
objectUnderTest.reportPasswd = "foo"; objectUnderTest.reportPasswd = "foo";
objectUnderTest.reportHost = "foo"; objectUnderTest.reportHost = "foo";
objectUnderTest.reportUser = "foo"; objectUnderTest.reportUser = "foo";
objectUnderTest.setCommand((byte)0); objectUnderTest.setCommand((byte) 0);
// Act // Act
final byte[] actual = objectUnderTest.toBytes(); final byte[] actual = objectUnderTest.toBytes();
// Assert result // Assert result
Assert.assertArrayEquals( Assert.assertArrayEquals(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 3, (byte) 102,
new byte[] {(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)3, (byte)102, (byte) 111, (byte) 111, (byte) 3, (byte) 102, (byte) 111, (byte) 111, (byte) 3, (byte) 102, (byte) 111,
(byte)111, (byte)111, (byte)3, (byte)102, (byte)111, (byte)111, (byte)3, (byte) 111, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0,
(byte)102, (byte)111, (byte)111, (byte)0, (byte)0, (byte)0, (byte)0, (byte) 0 }, actual);
(byte)0, (byte)0, (byte)0, (byte)0, (byte)0, (byte)0},
actual);
} }
// Test written by Diffblue Cover. // Test written by Diffblue Cover.
@@ -54,6 +53,6 @@ public class RegisterSlaveCommandPacketTest {
final byte[] actual = RegisterSlaveCommandPacket.toLH(n); final byte[] actual = RegisterSlaveCommandPacket.toLH(n);
// Assert result // Assert result
Assert.assertArrayEquals(new byte[] {(byte)0, (byte)0, (byte)0, (byte)0}, actual); Assert.assertArrayEquals(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0 }, actual);
} }
} }

1
example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientFlatMessageExample.java
View File

@@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector; import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.example.kafka.AbstractKafkaTest;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
/** /**

5
filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
View File

@@ -8,13 +8,12 @@ import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler; import org.apache.oro.text.regex.Perl5Compiler;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.collect.MapMaker; import com.google.common.cache.CacheBuilder;
import com.google.common.collect.MigrateMap; import com.google.common.collect.MigrateMap;
public class PatternUtils { public class PatternUtils {
@SuppressWarnings("deprecation") private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(CacheBuilder.newBuilder().softValues(),
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, Pattern>() { new Function<String, Pattern>() {
public Pattern apply(String pattern) { public Pattern apply(String pattern) {

1
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
View File

@@ -109,6 +109,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
super.start(); super.start();
} }
@SuppressWarnings("resource")
protected void initAlarmHandler() { protected void initAlarmHandler() {
logger.info("init alarmHandler begin..."); logger.info("init alarmHandler begin...");
String alarmHandlerClass = parameters.getAlarmHandlerClass(); String alarmHandlerClass = parameters.getAlarmHandlerClass();

5
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
View File

@@ -2,13 +2,12 @@ package com.alibaba.otter.canal.instance.spring;
import java.util.List; import java.util.List;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalMQConfig;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler; import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.instance.core.AbstractCanalInstance; import com.alibaba.otter.canal.instance.core.AbstractCanalInstance;
import com.alibaba.otter.canal.instance.core.CanalMQConfig;
import com.alibaba.otter.canal.meta.CanalMetaManager; import com.alibaba.otter.canal.meta.CanalMetaManager;
import com.alibaba.otter.canal.parse.CanalEventParser; import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry;
@@ -58,7 +57,7 @@ public class CanalInstanceWithSpring extends AbstractCanalInstance {
this.alarmHandler = alarmHandler; this.alarmHandler = alarmHandler;
} }
public void setMqConfig(CanalMQConfig mqConfig){ public void setMqConfig(CanalMQConfig mqConfig) {
this.mqConfig = mqConfig; this.mqConfig = mqConfig;
} }

12
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
View File

@@ -665,6 +665,14 @@ public class DatabaseTableMeta implements TableMetaTSDB {
this.fieldBlackFilterMap = fieldBlackFilterMap; this.fieldBlackFilterMap = fieldBlackFilterMap;
} }
public Map<String, List<String>> getFieldFilterMap() {
return fieldFilterMap;
}
public Map<String, List<String>> getFieldBlackFilterMap() {
return fieldBlackFilterMap;
}
public int getSnapshotInterval() { public int getSnapshotInterval() {
return snapshotInterval; return snapshotInterval;
} }
@@ -693,8 +701,4 @@ public class DatabaseTableMeta implements TableMetaTSDB {
return false; return false;
} }
public static void main(String[] args) {
String str = StringUtils.substringBefore("int(11)", "(");
System.out.println(str);
}
} }

26
prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/SinkCollector.java
View File

@@ -1,14 +1,8 @@
package com.alibaba.otter.canal.prometheus.impl; package com.alibaba.otter.canal.prometheus.impl;
import com.alibaba.otter.canal.instance.core.CanalInstance; import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST;
import com.alibaba.otter.canal.prometheus.InstanceRegistry;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
import com.google.common.base.Preconditions;
import io.prometheus.client.Collector; import io.prometheus.client.Collector;
import io.prometheus.client.CounterMetricFamily; import io.prometheus.client.CounterMetricFamily;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@@ -17,8 +11,14 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST; import org.slf4j.Logger;
import static java.util.concurrent.TimeUnit.NANOSECONDS; import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.prometheus.InstanceRegistry;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
import com.google.common.base.Preconditions;
/** /**
* @author Chuanyi Li * @author Chuanyi Li
@@ -31,9 +31,11 @@ public class SinkCollector extends Collector implements InstanceRegistry {
private static final String SINK_BLOCK_TIME_HELP = "Total sink blocking time in milliseconds"; private static final String SINK_BLOCK_TIME_HELP = "Total sink blocking time in milliseconds";
private final ConcurrentMap<String, SinkMetricsHolder> instances = new ConcurrentHashMap<String, SinkMetricsHolder>(); private final ConcurrentMap<String, SinkMetricsHolder> instances = new ConcurrentHashMap<String, SinkMetricsHolder>();
private SinkCollector() {} private SinkCollector(){
}
private static class SingletonHolder { private static class SingletonHolder {
private static final SinkCollector SINGLETON = new SinkCollector(); private static final SinkCollector SINGLETON = new SinkCollector();
} }
@@ -45,7 +47,8 @@ public class SinkCollector extends Collector implements InstanceRegistry {
public List<MetricFamilySamples> collect() { public List<MetricFamilySamples> collect() {
List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>(); List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME, CounterMetricFamily blockingCounter = new CounterMetricFamily(SINK_BLOCKING_TIME,
SINK_BLOCK_TIME_HELP, DEST_LABELS_LIST); SINK_BLOCK_TIME_HELP,
DEST_LABELS_LIST);
for (SinkMetricsHolder smh : instances.values()) { for (SinkMetricsHolder smh : instances.values()) {
blockingCounter.addMetric(smh.destLabelValues, (smh.eventsSinkBlockingTime.doubleValue() / NANO_PER_MILLI)); blockingCounter.addMetric(smh.destLabelValues, (smh.eventsSinkBlockingTime.doubleValue() / NANO_PER_MILLI));
} }
@@ -78,6 +81,7 @@ public class SinkCollector extends Collector implements InstanceRegistry {
} }
private class SinkMetricsHolder { private class SinkMetricsHolder {
private AtomicLong eventsSinkBlockingTime; private AtomicLong eventsSinkBlockingTime;
private List<String> destLabelValues; private List<String> destLabelValues;
} }